wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-14 12:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ray debugger stepping between tasks (#12075)

Browse Source
This commit is contained in:
Philipp Moritz
2020-12-06 21:50:18 -08:00
committed by GitHub
parent 260b07cf0c
commit 73a1a232b9
28 changed files with 267 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/common/task/task_spec.cc
+4
View File
@@ -193,6 +193,10 @@ const ResourceSet &TaskSpecification::GetRequiredPlacementResources() const {
return *required_placement_resources_;
}
std::string TaskSpecification::GetDebuggerBreakpoint() const {
return message_->debugger_breakpoint();
}
std::unordered_map<std::string, std::string>
TaskSpecification::OverrideEnvironmentVariables() const {
return MapFromProtobuf(message_->override_environment_variables());
src/ray/common/task/task_spec.h
+2
View File
@@ -131,6 +131,8 @@ class TaskSpecification : public MessageWrapper<rpc::TaskSpec> {
/// \return The recomputed dependencies for the task.
std::vector<rpc::ObjectReference> GetDependencies() const;
std::string GetDebuggerBreakpoint() const;
std::unordered_map<std::string, std::string> OverrideEnvironmentVariables() const;
bool IsDriverTask() const;
src/ray/common/task/task_util.h
+2
View File
@@ -87,6 +87,7 @@ class TaskSpecBuilder {
const std::unordered_map<std::string, double> &required_resources,
const std::unordered_map<std::string, double> &required_placement_resources,
const BundleID &bundle_id, bool placement_group_capture_child_tasks,
const std::string &debugger_breakpoint,
const std::unordered_map<std::string, std::string> &override_environment_variables =
{}) {
message_->set_type(TaskType::NORMAL_TASK);
@@ -108,6 +109,7 @@ class TaskSpecBuilder {
message_->set_placement_group_bundle_index(bundle_id.second);
message_->set_placement_group_capture_child_tasks(
placement_group_capture_child_tasks);
message_->set_debugger_breakpoint(debugger_breakpoint);
for (const auto &env : override_environment_variables) {
(*message_->mutable_override_environment_variables())[env.first] = env.second;
}
src/ray/core_worker/core_worker.cc
+8 -5
View File
@@ -39,14 +39,14 @@ void BuildCommonTaskSpec(
const std::unordered_map<std::string, double> &required_resources,
const std::unordered_map<std::string, double> &required_placement_resources,
std::vector<ObjectID> *return_ids, const ray::BundleID &bundle_id,
bool placement_group_capture_child_tasks,
bool placement_group_capture_child_tasks, const std::string debugger_breakpoint,
const std::unordered_map<std::string, std::string> &override_environment_variables) {
// Build common task spec.
builder.SetCommonTaskSpec(
task_id, name, function.GetLanguage(), function.GetFunctionDescriptor(), job_id,
current_task_id, task_index, caller_id, address, num_returns, required_resources,
required_placement_resources, bundle_id, placement_group_capture_child_tasks,
override_environment_variables);
debugger_breakpoint, override_environment_variables);
// Set task arguments.
for (const auto &arg : args) {
builder.AddArg(*arg);
@@ -1294,7 +1294,8 @@ void CoreWorker::SubmitTask(const RayFunction &function,
const TaskOptions &task_options,
std::vector<ObjectID> *return_ids, int max_retries,
BundleID placement_options,
bool placement_group_capture_child_tasks) {
bool placement_group_capture_child_tasks,
const std::string &debugger_breakpoint) {
TaskSpecBuilder builder;
const int next_task_index = worker_context_.GetNextTaskIndex();
const auto task_id =
@@ -1320,7 +1321,7 @@ void CoreWorker::SubmitTask(const RayFunction &function,
rpc_address_, function, args, task_options.num_returns,
constrained_resources, required_resources, return_ids,
placement_options, placement_group_capture_child_tasks,
override_environment_variables);
debugger_breakpoint, override_environment_variables);
TaskSpecification task_spec = builder.Build();
if (options_.is_local_mode) {
ExecuteTaskLocalMode(task_spec);
@@ -1376,6 +1377,7 @@ Status CoreWorker::CreateActor(const RayFunction &function,
new_placement_resources, &return_ids,
actor_creation_options.placement_options,
actor_creation_options.placement_group_capture_child_tasks,
"", /* debugger_breakpoint */
override_environment_variables);
builder.SetActorCreationTaskSpec(actor_id, actor_creation_options.max_restarts,
actor_creation_options.dynamic_worker_options,
@@ -1504,6 +1506,7 @@ void CoreWorker::SubmitActorTask(const ActorID &actor_id, const RayFunction &fun
required_resources, return_ids,
std::make_pair(PlacementGroupID::Nil(), -1),
true, /* placement_group_capture_child_tasks */
"", /* debugger_breakpoint */
override_environment_variables);
// NOTE: placement_group_capture_child_tasks and override_environment_variables will be
// ignored in the actor because we should always follow the actor's option.
@@ -1802,7 +1805,7 @@ Status CoreWorker::ExecuteTask(const TaskSpecification &task_spec,
status = options_.task_execution_callback(
task_type, task_spec.GetName(), func,
task_spec.GetRequiredResources().GetResourceMap(), args, arg_reference_ids,
return_ids, return_objects);
return_ids, task_spec.GetDebuggerBreakpoint(), return_objects);
absl::optional<rpc::Address> caller_address(
options_.is_local_mode ? absl::optional<rpc::Address>()
src/ray/core_worker/core_worker.h
+5 -2
View File
@@ -61,7 +61,7 @@ struct CoreWorkerOptions {
const std::unordered_map<std::string, double> &required_resources,
const std::vector<std::shared_ptr<RayObject>> &args,
const std::vector<ObjectID> &arg_reference_ids,
const std::vector<ObjectID> &return_ids,
const std::vector<ObjectID> &return_ids, const std::string &debugger_breakpoint,
std::vector<std::shared_ptr<RayObject>> *results)>;
CoreWorkerOptions()
@@ -632,12 +632,15 @@ class CoreWorker : public rpc::CoreWorkerServiceHandler {
/// \param[in] max_retires max number of retry when the task fails.
/// \param[in] placement_options placement group options.
/// \param[in] placement_group_capture_child_tasks whether or not the submitted task
/// \param[in] debugger_breakpoint breakpoint to drop into for the debugger after this
/// task starts executing, or "" if we do not want to drop into the debugger.
/// should capture parent's placement group implicilty.
void SubmitTask(const RayFunction &function,
const std::vector<std::unique_ptr<TaskArg>> &args,
const TaskOptions &task_options, std::vector<ObjectID> *return_ids,
int max_retries, BundleID placement_options,
bool placement_group_capture_child_tasks);
bool placement_group_capture_child_tasks,
const std::string &debugger_breakpoint);
/// Create an actor.
///
src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc
+1 -1
View File
@@ -110,7 +110,7 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(
const std::unordered_map<std::string, double> &required_resources,
const std::vector<std::shared_ptr<ray::RayObject>> &args,
const std::vector<ObjectID> &arg_reference_ids,
const std::vector<ObjectID> &return_ids,
const std::vector<ObjectID> &return_ids, const std::string &debugger_breakpoint,
std::vector<std::shared_ptr<ray::RayObject>> *results) {
JNIEnv *env = GetJNIEnv();
RAY_CHECK(java_task_executor);
src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc
+3 -1
View File
@@ -222,7 +222,9 @@ JNIEXPORT jobject JNICALL Java_io_ray_runtime_task_NativeTaskSubmitter_nativeSub
ray_function, task_args, task_options, &return_ids,
/*max_retries=*/0,
/*placement_options=*/
std::pair<ray::PlacementGroupID, int64_t>(ray::PlacementGroupID::Nil(), 0), true);
std::pair<ray::PlacementGroupID, int64_t>(ray::PlacementGroupID::Nil(), 0),
/*placement_group_capture_child_tasks=*/true,
/*debugger_breakpoint*/ "");
// This is to avoid creating an empty java list and boost performance.
if (return_ids.empty()) {
src/ray/core_worker/test/core_worker_test.cc
+3 -2
View File
@@ -257,7 +257,8 @@ void CoreWorkerTest::TestNormalTask(std::unordered_map<std::string, double> &res
TaskOptions options;
std::vector<ObjectID> return_ids;
driver.SubmitTask(func, args, options, &return_ids, /*max_retries=*/0,
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true,
/*debugger_breakpoint=*/"");
ASSERT_EQ(return_ids.size(), 1);
@@ -533,7 +534,7 @@ TEST_F(ZeroNodeTest, TestTaskSpecPerf) {
builder.SetCommonTaskSpec(RandomTaskId(), options.name, function.GetLanguage(),
function.GetFunctionDescriptor(), job_id, RandomTaskId(), 0,
RandomTaskId(), address, num_returns, resources, resources,
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
// Set task arguments.
for (const auto &arg : args) {
builder.AddArg(*arg);
src/ray/core_worker/test/direct_task_transport_test.cc
+1 -1
View File
@@ -328,7 +328,7 @@ TaskSpecification BuildTaskSpec(const std::unordered_map<std::string, double> &r
builder.SetCommonTaskSpec(TaskID::Nil(), "dummy_task", Language::PYTHON,
function_descriptor, JobID::Nil(), TaskID::Nil(), 0,
TaskID::Nil(), empty_address, 1, resources, resources,
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
return builder.Build();
}
src/ray/core_worker/test/mock_worker.cc
+2 -1
View File
@@ -46,7 +46,7 @@ class MockWorker {
options.node_manager_port = node_manager_port;
options.raylet_ip_address = "127.0.0.1";
options.task_execution_callback =
std::bind(&MockWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8);
std::bind(&MockWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8, _9);
options.ref_counting_enabled = true;
options.num_workers = 1;
options.metrics_agent_port = -1;
@@ -62,6 +62,7 @@ class MockWorker {
const std::vector<std::shared_ptr<RayObject>> &args,
const std::vector<ObjectID> &arg_reference_ids,
const std::vector<ObjectID> &return_ids,
const std::string &debugger_breakpoint,
std::vector<std::shared_ptr<RayObject>> *results) {
// Note that this doesn't include dummy object id.
const ray::FunctionDescriptor function_descriptor =
src/ray/gcs/test/gcs_test_util.h
+1 -1
View File
@@ -41,7 +41,7 @@ struct Mocker {
builder.SetCommonTaskSpec(task_id, name + ":" + empty_descriptor->CallString(),
Language::PYTHON, empty_descriptor, job_id, TaskID::Nil(),
0, TaskID::Nil(), owner_address, 1, resource, resource,
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
builder.SetActorCreationTaskSpec(actor_id, max_restarts, {}, 1, detached, name);
return builder.Build();
}
src/ray/protobuf/common.proto
+3
View File
@@ -200,6 +200,9 @@ message TaskSpec {
// the receiver will not execute the task. This field is used by async actors
// to guarantee task submission order after restart.
bool skip_execution = 22;
// Breakpoint if this task should drop into the debugger when it starts executing
// and "" if the task should not drop into the debugger.
bytes debugger_breakpoint = 23;
}
message Bundle {
src/ray/raylet/scheduling/cluster_task_manager_test.cc
+1 -1
View File
@@ -80,7 +80,7 @@ Task CreateTask(const std::unordered_map<std::string, double> &required_resource
FunctionDescriptorBuilder::BuildPython("", "", "", ""),
job_id, TaskID::Nil(), 0, TaskID::Nil(), address, 0,
required_resources, {},
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
for (int i = 0; i < num_args; i++) {
ObjectID put_id = ObjectID::FromIndex(TaskID::Nil(), /*index=*/i + 1);
src/ray/raylet/task_dependency_manager_test.cc
+1 -1
View File
@@ -69,7 +69,7 @@ static inline Task ExampleTask(const std::vector<ObjectID> &arguments,
FunctionDescriptorBuilder::BuildPython("", "", "", ""),
JobID::Nil(), RandomTaskId(), 0, RandomTaskId(), address,
num_returns, {}, {},
std::make_pair(PlacementGroupID::Nil(), -1), true);
std::make_pair(PlacementGroupID::Nil(), -1), true, "");
builder.SetActorCreationTaskSpec(ActorID::Nil(), 1, {}, 1, false, "", false);
for (const auto &arg : arguments) {
builder.AddArg(TaskArgByReference(arg, rpc::Address()));