wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

[xray] Integrate worker.py with raylet. (#1810)

Browse Source 
* Integrate worker with raylet.

* Begin allowing worker to attach to cluster.

* Fix linting and documentation.

* Fix linting.

* Comment tests back in.

* Fix type of worker command.

* Remove xray python files and tests.

* Fix from rebase.

* Add test.

* Copy over raylet executable.

* Small cleanup.
This commit is contained in:
Robert Nishihara
2018-04-03 02:38:56 -07:00
committed by Philipp Moritz
parent 0fc989c6c1
commit fbfbb1c079
22 changed files with 459 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/gcs/CMakeLists.txt
+9
View File
@@ -19,6 +19,15 @@ add_custom_command(
add_custom_target(gen_gcs_fbs DEPENDS ${GCS_FBS_OUTPUT_FILES})
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_gcs_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${GCS_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${GCS_FBS_SRC}"
VERBATIM)
ADD_RAY_TEST(client_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
ADD_RAY_TEST(asio_test STATIC_LINK_LIBS ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} gtest gtest_main pthread ${Boost_SYSTEM_LIBRARY})
src/ray/python/default_worker.py
-18
View File
@@ -1,18 +0,0 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
from worker import Worker
parser = argparse.ArgumentParser()
parser.add_argument("raylet_socket_name")
parser.add_argument("object_store_socket_name")
if __name__ == '__main__':
args = parser.parse_args()
worker = Worker(args.raylet_socket_name, args.object_store_socket_name,
is_worker=True)
worker.main_loop()
src/ray/python/one_test_driver.py
-33
View File
@@ -1,33 +0,0 @@
import argparse
import ray
from worker import Worker, logger
from ray.utils import random_string
parser = argparse.ArgumentParser()
parser.add_argument("raylet_socket_name")
parser.add_argument("object_store_socket_name")
if __name__ == '__main__':
args = parser.parse_args()
driver = Worker(args.raylet_socket_name, args.object_store_socket_name,
is_worker=False)
task1 = ray.local_scheduler.Task(
ray.local_scheduler.ObjectID(random_string()),
ray.local_scheduler.ObjectID(random_string()),
[],
1,
ray.local_scheduler.ObjectID(random_string()),
0)
logger.debug("submitting", task1.task_id())
driver.node_manager_client.submit(task1)
logger.debug("Return values were", task1.returns())
print("[DRIVER] Return values were", task1.returns())
# Make sure the tasks get executed and we can get the result of the
# last task
obj = driver.get(task1.returns(), timeout_ms=1000)
print("[DRIVER]: task1 driver.get result ", obj)
src/ray/python/test_driver.py
-40
View File
@@ -1,40 +0,0 @@
import argparse
import ray
from worker import Worker, logger
from ray.utils import random_string
parser = argparse.ArgumentParser()
parser.add_argument("raylet_socket_name")
parser.add_argument("object_store_socket_name")
if __name__ == '__main__':
args = parser.parse_args()
driver = Worker(args.raylet_socket_name, args.object_store_socket_name,
is_worker=False)
task = ray.local_scheduler.Task(
ray.local_scheduler.ObjectID(random_string()),
ray.local_scheduler.ObjectID(random_string()),
[],
1,
ray.local_scheduler.ObjectID(random_string()),
0)
logger.debug("submitting %s", task.task_id())
driver.node_manager_client.submit(task)
logger.debug("Return values were %s", task.returns())
task2 = ray.local_scheduler.Task(
ray.local_scheduler.ObjectID(random_string()),
ray.local_scheduler.ObjectID(random_string()),
task.returns(),
1,
ray.local_scheduler.ObjectID(random_string()),
0)
logger.debug("Submitting dependent task 2 %s", task2.task_id())
driver.node_manager_client.submit(task2)
# Make sure the tasks get executed and we can get the result of the last
# task.
obj = driver.get(task2.returns(), timeout_ms=1000)
src/ray/python/test_driver_taskchains.py
-115
View File
@@ -1,115 +0,0 @@
import argparse
import ray
from worker import Worker, logger
from ray.utils import random_string
parser = argparse.ArgumentParser()
parser.add_argument("raylet_socket_name")
parser.add_argument("object_store_socket_name")
def submit_task_withdep(driver_handle, task_object_dependencies=[]):
''' submit a task that depend on a list of @args'''
task = ray.local_scheduler.Task(
ray.local_scheduler.ObjectID(random_string()),
ray.local_scheduler.ObjectID(random_string()),
task_object_dependencies,
1, # num_returns
ray.local_scheduler.ObjectID(random_string()),
0)
logger.debug("[DRIVER]: submitting task ", task.task_id())
driver_handle.node_manager_client.submit(task)
logger.debug("[DRIVER]: task return values", task.returns())
return task.returns()
def submit_tasks_nodep(driver_handle, num_tasks):
''' submit a task that depend on a list of @args'''
for i in range(num_tasks):
task = ray.local_scheduler.Task(
ray.local_scheduler.ObjectID(random_string()),
ray.local_scheduler.ObjectID(random_string()),
[],
1, # num_returns
ray.local_scheduler.ObjectID(random_string()),
0)
logger.debug("[DRIVER]: submitting task ", task.task_id())
driver_handle.node_manager_client.submit(task)
logger.debug("[DRIVER]: task return values", task.returns())
def submit_task_chains(num_chains, tasks_per_chain):
# return task placement map on output
chain_returns = []
task_placement_map_ = {}
for chain_num in range(num_chains):
last_task_returns = []
task_placement_map_[chain_num] = []
for i in range(tasks_per_chain):
task_returns = submit_task_withdep(
driver,
task_object_dependencies=last_task_returns)
last_task_returns = task_returns
task_placement_map_[chain_num].append(task_returns[0])
chain_returns.append(last_task_returns)
logger.debug("chain_returns=", chain_returns)
chain_results = driver.get([r[0] for r in chain_returns], timeout_ms=5000)
print("[DRIVER]: chain return values: ", chain_results)
return task_placement_map_
def TEST_run_task_chains(num_chains, tasks_per_chain):
task_placement_map = submit_task_chains(num_chains=num_chains,
tasks_per_chain=tasks_per_chain)
logger.debug("[DRIVER]: task placement information, per chain:")
task_placement_total = []
for chain_num in range(len(task_placement_map)):
task_placement_list = driver.get(task_placement_map[chain_num],
timeout_ms=5000)
task_placement_total += [t[1] for t in task_placement_list]
logger.debug(chain_num, task_placement_list)
logger.debug("task placement overall: ", task_placement_total)
task_placement_stats = [(v, task_placement_total.count(v))
for v in set(task_placement_total)]
num_total_tasks = sum([t[1] for t in task_placement_stats])
print("total tasks executed = ", num_total_tasks)
assert(num_total_tasks == num_chains * tasks_per_chain)
print("task placement breakdown: total=", task_placement_stats)
def TEST_run_tasks_nodep(num_tasks):
# This test is the same as having num_tasks chains with 1 task per chain
# In this test we assume the num_tasks x 1 chain structure.
task_placement_map = submit_task_chains(num_chains=num_tasks,
tasks_per_chain=1)
logger.debug("[DRIVER]: task placement information, per chain:")
task_placement_total = []
for chain_num in range(len(task_placement_map)):
task_placement_list = driver.get(task_placement_map[chain_num],
timeout_ms=5000)
task_placement_total += [t[1] for t in task_placement_list]
logger.debug(chain_num, task_placement_list)
logger.debug("task placement overall: ", task_placement_total)
task_placement_stats = [(v, task_placement_total.count(v)) for v in
set(task_placement_total)]
num_total_tasks = sum([t[1] for t in task_placement_stats])
print("total tasks executed = ", num_total_tasks)
assert(num_total_tasks == num_tasks)
print("task placement breakdown: total=", task_placement_stats)
if __name__ == '__main__':
args = parser.parse_args()
driver = Worker(args.raylet_socket_name, args.object_store_socket_name,
is_worker=False)
# Set up the experiment : number of chains and tasks per chain.
# TEST_run_task_chains(num_chains=10, tasks_per_chain=100)
TEST_run_tasks_nodep(10000)
src/ray/python/worker.py
-67
View File
@@ -1,67 +0,0 @@
import logging
import ray
import pyarrow
import pyarrow.plasma as plasma
from ray.utils import random_string
logging.basicConfig()
logger = logging.getLogger(__name__)
# The default return value to put in the object store.
RETURN_VALUE = 0
class Worker(object):
total_task_count = 0
def __init__(self, raylet_socket_name, object_store_socket_name,
is_worker):
# Connect to the Raylet and object store.
self.node_manager_client = ray.local_scheduler.LocalSchedulerClient(
raylet_socket_name, random_string(), is_worker)
self.plasma_client = plasma.connect(object_store_socket_name, "", 0)
self.serialization_context = pyarrow.default_serialization_context()
self.raylet_socket_name = raylet_socket_name
self.object_store_socket_name = object_store_socket_name
def main_loop(self):
while True:
self.get_task()
def get(self, object_ids, timeout_ms=-1):
for object_id in object_ids:
self.node_manager_client.reconstruct_object(object_id.id())
plasma_ids = [plasma.ObjectID(argument.id()) for argument in
object_ids]
values = self.plasma_client.get(plasma_ids, timeout_ms,
self.serialization_context)
assert(all(value[0] == RETURN_VALUE for value in values))
return values
def get_task(self):
logger.debug("[WORKER] waiting for task")
task = self.node_manager_client.get_task()
logger.debug("Worker assigned %s with arguments %s",
ray.utils.binary_to_hex(task.task_id().id()),
" ".join([ray.utils.binary_to_hex(argument.id()) for
argument in task.arguments()]))
# Get the arguments. NOTE(swang): This will hang forever if the
# arguments have been evicted.
arguments = self.get(task.arguments())
for object_id in task.returns():
self.plasma_client.put((RETURN_VALUE, self.raylet_socket_name),
plasma.ObjectID(object_id.id()))
objval = self.plasma_client.get([plasma.ObjectID(object_id.id())])
assert(all([o[0] == RETURN_VALUE for o in objval]))
logger.debug("Worker returned %s",
" ".join([ray.utils.binary_to_hex(return_id.id()) for
return_id in task.returns()]))
# Release the arguments.
del arguments
src/ray/raylet/main.cc
+10 -6
View File
@@ -5,13 +5,14 @@
#ifndef RAYLET_TEST
int main(int argc, char *argv[]) {
RAY_CHECK(argc == 6);
RAY_CHECK(argc == 7);
const std::string raylet_socket_name = std::string(argv[1]);
const std::string store_socket_name = std::string(argv[2]);
const std::string node_ip_address = std::string(argv[3]);
const std::string redis_address = std::string(argv[4]);
int redis_port = std::stoi(argv[5]);
const std::string worker_command = std::string(argv[6]);
// Configuration for the node manager.
ray::raylet::NodeManagerConfig node_manager_config;
@@ -21,11 +22,13 @@ int main(int argc, char *argv[]) {
ray::raylet::ResourceSet(std::move(static_resource_conf));
node_manager_config.num_initial_workers = 0;
// Use a default worker that can execute empty tasks with dependencies.
node_manager_config.worker_command.push_back("python");
node_manager_config.worker_command.push_back(
"../../../src/ray/python/default_worker.py");
node_manager_config.worker_command.push_back(raylet_socket_name.c_str());
node_manager_config.worker_command.push_back(store_socket_name.c_str());
std::stringstream worker_command_stream(worker_command);
std::string token;
while (getline(worker_command_stream, token, ' ')) {
node_manager_config.worker_command.push_back(token);
}
// TODO(swang): Set this from a global config.
node_manager_config.heartbeat_period_ms = 100;
@@ -41,6 +44,7 @@ int main(int argc, char *argv[]) {
// Initialize the node manager.
boost::asio::io_service main_service;
std::unique_ptr<boost::asio::io_service> object_manager_service;
object_manager_service.reset(new boost::asio::io_service());
ray::raylet::Raylet server(main_service, std::move(object_manager_service),
raylet_socket_name, node_ip_address, redis_address,
src/ray/raylet/node_manager.h
+1 -1
View File
@@ -21,7 +21,7 @@ namespace raylet {
struct NodeManagerConfig {
ResourceSet resource_config;
int num_initial_workers;
std::vector<const char *> worker_command;
std::vector<std::string> worker_command;
uint64_t heartbeat_period_ms;
};
src/ray/raylet/raylet.cc
+2 -2
View File
@@ -69,8 +69,8 @@ ray::Status Raylet::RegisterGcs(const std::string &node_ip_address,
client_info.resources_total_capacity.push_back(resource_pair.second);
}
RAY_LOG(DEBUG) << "NM LISTENING ON: IP " << client_info.node_manager_address << " PORT "
<< client_info.node_manager_port;
RAY_LOG(DEBUG) << "Node manager listening on: IP " << client_info.node_manager_address
<< " port " << client_info.node_manager_port;
RAY_RETURN_NOT_OK(gcs_client_->client_table().Connect(client_info));
auto node_manager_client_added = [this](gcs::AsyncGcsClient *client, const UniqueID &id,
src/ray/raylet/worker_pool.cc
+11 -4
View File
@@ -8,9 +8,8 @@ namespace ray {
namespace raylet {
/// A constructor that initializes a worker pool with num_workers workers.
WorkerPool::WorkerPool(int num_workers, const std::vector<const char *> &worker_command)
WorkerPool::WorkerPool(int num_workers, const std::vector<std::string> &worker_command)
: worker_command_(worker_command) {
worker_command_.push_back(NULL);
// Ignore SIGCHLD signals. If we don't do this, then worker processes will
// become zombies instead of dying gracefully.
signal(SIGCHLD, SIG_IGN);
@@ -37,9 +36,17 @@ void WorkerPool::StartWorker() {
// Reset the SIGCHLD handler for the worker.
signal(SIGCHLD, SIG_DFL);
// Try to execute the worker command.
int rv = execvp(worker_command_[0], (char *const *)worker_command_.data());
// Extract pointers from the worker command to pass into execvp.
std::vector<const char *> worker_command_args;
for (auto const &token : worker_command_) {
worker_command_args.push_back(token.c_str());
}
worker_command_args.push_back(nullptr);
// Try to execute the worker command.
int rv = execvp(worker_command_args[0],
const_cast<char *const *>(worker_command_args.data()));
// The worker failed to start. This is a fatal error.
RAY_LOG(FATAL) << "Failed to start worker with return value " << rv;
}
src/ray/raylet/worker_pool.h
+2 -2
View File
@@ -25,7 +25,7 @@ class WorkerPool {
/// pool.
///
/// \param num_workers The number of workers to start.
WorkerPool(int num_workers, const std::vector<const char *> &worker_command);
WorkerPool(int num_workers, const std::vector<std::string> &worker_command);
/// Destructor responsible for freeing a set of workers owned by this class.
~WorkerPool();
@@ -74,7 +74,7 @@ class WorkerPool {
std::shared_ptr<Worker> PopWorker();
private:
std::vector<const char *> worker_command_;
std::vector<std::string> worker_command_;
/// The pool of idle workers.
std::list<std::shared_ptr<Worker>> pool_;
/// All workers that have registered and are still connected, including both
src/ray/test/run_task_test.sh
-23
View File
@@ -1,23 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/python/ray/core
# Cause the script to exit if a single command fails.
set -e
set -x
# Tear down the Raylet.
#bash ../../../src/ray/test/stop_raylets.sh
# Set up a single Raylet.
bash ../../../src/ray/test/start_raylets.sh
sleep 1
# Connect a driver to the raylet and make sure it completes.
python ../../../src/ray/python/test_driver.py /tmp/raylet1 /tmp/store1
sleep 1
./src/common/thirdparty/redis/src/redis-cli -p 6379 shutdown
bash ../../../src/ray/test/stop_raylets.sh
src/ray/test/start_redis.sh
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
# This needs to be run in the build tree, which is normally ray/python/ray/core
# Cause the script to exit if a single command fails.
set -e
# Start the GCS.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 >/dev/null &
sleep 1s
src/ray/test/stop_raylets.sh
-9
View File
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
killall raylet
sleep 1
killall plasma_store
sleep 1
killall redis-server
sleep 1
rm /tmp/store* /tmp/raylet*