wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 16:46:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

YAPF, take 3 (#2098)

Browse Source 
* Use pep8 style

The original style file is actually just pep8 style, but with everything
spelled out. It's easier to use the `based_on_style` feature. Any overrides are
clearer that way.

* Improve yapf script

1. Do formatting in parallel
2. Lint RLlib
3. Use .style.yapf file

* Pull out expressions into variables

* Don't format rllib

* Don't allow splits in dicts

* Apply yapf

* Disallow single line if-statements

* Use arithmetic comparison

* Simplify checking for changed files

* Pull out expr into var
This commit is contained in:
Alok Singh
2018-05-19 16:07:28 -07:00
committed by Richard Liaw
parent 8e0962bb9c
commit 9a8f29e571
14 changed files with 218 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.style.yapf
+3 -188
View File
@@ -1,189 +1,4 @@
[style] [style]
# Align closing bracket with visual indentation. based_on_style=pep8
align_closing_bracket_with_visual_indent=True allow_split_before_dict_value=False
join_multiple_lines=False
# Allow dictionary keys to exist on multiple lines. For example:
#
# x = {
# ('this is the first element of a tuple',
# 'this is the second element of a tuple'):
# value,
# }
allow_multiline_dictionary_keys=False
# Allow lambdas to be formatted on more than one line.
allow_multiline_lambdas=False
# Insert a blank line before a class-level docstring.
blank_line_before_class_docstring=False
# Insert a blank line before a 'def' or 'class' immediately nested
# within another 'def' or 'class'. For example:
#
# class Foo:
# # <------ this blank line
# def method():
# ...
blank_line_before_nested_class_or_def=False
# Do not split consecutive brackets. Only relevant when
# dedent_closing_brackets is set. For example:
#
# call_func_that_takes_a_dict(
# {
# 'key1': 'value1',
# 'key2': 'value2',
# }
# )
#
# would reformat to:
#
# call_func_that_takes_a_dict({
# 'key1': 'value1',
# 'key2': 'value2',
# })
coalesce_brackets=True
# The column limit.
column_limit=79
# Indent width used for line continuations.
continuation_indent_width=4
# Put closing brackets on a separate line, dedented, if the bracketed
# expression can't fit in a single line. Applies to all kinds of brackets,
# including function definitions and calls. For example:
#
# config = {
# 'key1': 'value1',
# 'key2': 'value2',
# } # <--- this bracket is dedented and on a separate line
#
# time_series = self.remote_client.query_entity_counters(
# entity='dev3246.region1',
# key='dns.query_latency_tcp',
# transform=Transformation.AVERAGE(window=timedelta(seconds=60)),
# start_ts=now()-timedelta(days=3),
# end_ts=now(),
# ) # <--- this bracket is dedented and on a separate line
dedent_closing_brackets=False
# Place each dictionary entry onto its own line.
each_dict_entry_on_separate_line=True
# The regex for an i18n comment. The presence of this comment stops
# reformatting of that line, because the comments are required to be
# next to the string they translate.
i18n_comment=
# The i18n function call names. The presence of this function stops
# reformattting on that line, because the string it has cannot be moved
# away from the i18n comment.
i18n_function_call=
# Indent the dictionary value if it cannot fit on the same line as the
# dictionary key. For example:
#
# config = {
# 'key1':
# 'value1',
# 'key2': value1 +
# value2,
# }
indent_dictionary_value=True
# The number of columns to use for indentation.
indent_width=4
# Join short lines into one line. E.g., single line 'if' statements.
join_multiple_lines=True
# Do not include spaces around selected binary operators. For example:
#
# 1 + 2 * 3 - 4 / 5
#
# will be formatted as follows when configured with a value "*,/":
#
# 1 + 2*3 - 4/5
#
no_spaces_around_selected_binary_operators=set([])
# Use spaces around default or named assigns.
spaces_around_default_or_named_assign=False
# Use spaces around the power operator.
spaces_around_power_operator=False
# The number of spaces required before a trailing comment.
spaces_before_comment=2
# Insert a space between the ending comma and closing bracket of a list,
# etc.
space_between_ending_comma_and_closing_bracket=True
# Split before arguments if the argument list is terminated by a
# comma.
split_arguments_when_comma_terminated=False
# Set to True to prefer splitting before '&', '|' or '^' rather than
# after.
split_before_bitwise_operator=True
# Split before a dictionary or set generator (comp_for). For example, note
# the split before the 'for':
#
# foo = {
# variable: 'Hello world, have a nice day!'
# for variable in bar if variable != 42
# }
split_before_dict_set_generator=True
# If an argument / parameter list is going to be split, then split before
# the first argument.
split_before_first_argument=False
# Set to True to prefer splitting before 'and' or 'or' rather than
# after.
split_before_logical_operator=True
# Split named assignments onto individual lines.
split_before_named_assigns=True
# The penalty for splitting right after the opening bracket.
split_penalty_after_opening_bracket=30
# The penalty for splitting the line after a unary operator.
split_penalty_after_unary_operator=10000
# The penalty for splitting right before an if expression.
split_penalty_before_if_expr=0
# The penalty of splitting the line around the '&', '|', and '^'
# operators.
split_penalty_bitwise_operator=300
# The penalty for characters over the column limit.
split_penalty_excess_character=4500
# The penalty incurred by adding a line split to the unwrapped line. The
# more line splits added the higher the penalty.
split_penalty_for_added_line_split=30
# The penalty of splitting a list of "import as" names. For example:
#
# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1,
# long_argument_2,
# long_argument_3)
#
# would reformat to something like:
#
# from a_very_long_or_indented_module_name_yada_yad import (
# long_argument_1, long_argument_2, long_argument_3)
split_penalty_import_names=0
# The penalty of splitting the line around the 'and' and 'or'
# operators.
split_penalty_logical_operator=300
# Use the Tab character for indentation.
use_tabs=False
.travis/yapf.sh
+21 -19
View File
@@ -1,27 +1,29 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Cause the script to exit if a single command fails # Cause the script to exit if a single command fails
set -e set -eo pipefail
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) # this stops git rev-parse from failing if we run this from the .git directory
builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
pushd $ROOT_DIR/../test ROOT="$(git rev-parse --show-toplevel)"
find . -name '*.py' -type f -exec yapf --style=pep8 -i -r {} \; builtin cd "$ROOT"
popd
pushd $ROOT_DIR/../python yapf \
find . -name '*.py' -type f -not -path './ray/dataframe/*' -not -path './ray/rllib/*' -not -path './ray/cloudpickle/*' -exec yapf --style=pep8 -i -r {} \; --style "$ROOT/.style.yapf" \
popd --in-place --recursive --parallel \
--exclude 'python/ray/cloudpickle' \
--exclude 'python/ray/dataframe' \
--exclude 'python/ray/rllib' \
-- \
'test' 'python'
CHANGED_FILES=(`git diff --name-only`) if ! git diff --quiet; then
if [ "$CHANGED_FILES" ]; then echo 'Reformatted staged files. Please review and stage the changes.'
echo 'Reformatted staged files. Please review and stage the changes.' echo 'Files updated:'
echo echo
echo 'Files updated:'
for file in ${CHANGED_FILES[@]}; do git --no-pager diff --name-only
echo " $file"
done exit 1
exit 1
else
exit 0
fi fi
python/ray/actor.py
+14 -26
View File
@@ -857,32 +857,20 @@ class ActorHandle(object):
A dictionary of the information needed to reconstruct the object. A dictionary of the information needed to reconstruct the object.
""" """
state = { state = {
"actor_id": "actor_id": self._ray_actor_id.id(),
self._ray_actor_id.id(), "class_name": self._ray_class_name,
"class_name": "actor_forks": self._ray_actor_forks,
self._ray_class_name, "actor_cursor": self._ray_actor_cursor.id(),
"actor_forks": "actor_counter": 0, # Reset the actor counter.
self._ray_actor_forks, "actor_method_names": self._ray_actor_method_names,
"actor_cursor": "method_signatures": self._ray_method_signatures,
self._ray_actor_cursor.id(), "method_num_return_vals": self._ray_method_num_return_vals,
"actor_counter": "actor_creation_dummy_object_id": self.
0, # Reset the actor counter. _ray_actor_creation_dummy_object_id.id(),
"actor_method_names": "actor_method_cpus": self._ray_actor_method_cpus,
self._ray_actor_method_names, "actor_driver_id": self._ray_actor_driver_id.id(),
"method_signatures": "previous_actor_handle_id": self._ray_actor_handle_id.id(),
self._ray_method_signatures, "ray_forking": ray_forking
"method_num_return_vals":
self._ray_method_num_return_vals,
"actor_creation_dummy_object_id":
self._ray_actor_creation_dummy_object_id.id(),
"actor_method_cpus":
self._ray_actor_method_cpus,
"actor_driver_id":
self._ray_actor_driver_id.id(),
"previous_actor_handle_id":
self._ray_actor_handle_id.id(),
"ray_forking":
ray_forking
} }
if ray_forking: if ray_forking:
python/ray/autoscaler/autoscaler.py
+4 -8
View File
@@ -182,19 +182,15 @@ class LoadMetrics(object):
nodes_used += max_frac nodes_used += max_frac
idle_times = [now - t for t in self.last_used_time_by_ip.values()] idle_times = [now - t for t in self.last_used_time_by_ip.values()]
return { return {
"ResourceUsage": "ResourceUsage": ", ".join([
", ".join([
"{}/{} {}".format( "{}/{} {}".format(
round(resources_used[rid], 2), round(resources_used[rid], 2),
round(resources_total[rid], 2), rid) round(resources_total[rid], 2), rid)
for rid in sorted(resources_used) for rid in sorted(resources_used)
]), ]),
"NumNodesConnected": "NumNodesConnected": len(self.static_resources_by_ip),
len(self.static_resources_by_ip), "NumNodesUsed": round(nodes_used, 2),
"NumNodesUsed": "NodeIdleSeconds": "Min={} Mean={} Max={}".format(
round(nodes_used, 2),
"NodeIdleSeconds":
"Min={} Mean={} Max={}".format(
int(np.min(idle_times)) if idle_times else -1, int(np.min(idle_times)) if idle_times else -1,
int(np.mean(idle_times)) if idle_times else -1, int(np.mean(idle_times)) if idle_times else -1,
int(np.max(idle_times)) if idle_times else -1), int(np.max(idle_times)) if idle_times else -1),
python/ray/autoscaler/aws/config.py
+15 -16
View File
@@ -209,22 +209,21 @@ def _configure_security_group(config):
assert security_group, "Failed to create security group" assert security_group, "Failed to create security group"
if not security_group.ip_permissions: if not security_group.ip_permissions:
security_group.authorize_ingress( security_group.authorize_ingress(IpPermissions=[{
IpPermissions=[{ "FromPort": -1,
"FromPort": -1, "ToPort": -1,
"ToPort": -1, "IpProtocol": "-1",
"IpProtocol": "-1", "UserIdGroupPairs": [{
"UserIdGroupPairs": [{ "GroupId": security_group.id
"GroupId": security_group.id }]
}] }, {
}, { "FromPort": 22,
"FromPort": 22, "ToPort": 22,
"ToPort": 22, "IpProtocol": "TCP",
"IpProtocol": "TCP", "IpRanges": [{
"IpRanges": [{ "CidrIp": "0.0.0.0/0"
"CidrIp": "0.0.0.0/0" }]
}] }])
}])
if "SecurityGroupIds" not in config["head_node"]: if "SecurityGroupIds" not in config["head_node"]:
print("SecurityGroupIds not specified for head node, using {}".format( print("SecurityGroupIds not specified for head node, using {}".format(
python/ray/autoscaler/aws/node_provider.py
+3 -6
View File
@@ -101,12 +101,9 @@ class AWSNodeProvider(NodeProvider):
"Value": v, "Value": v,
}) })
conf.update({ conf.update({
"MinCount": "MinCount": 1,
1, "MaxCount": count,
"MaxCount": "TagSpecifications": conf.get("TagSpecifications", []) + [{
count,
"TagSpecifications":
conf.get("TagSpecifications", []) + [{
"ResourceType": "instance", "ResourceType": "instance",
"Tags": tag_pairs, "Tags": tag_pairs,
}] }]
python/ray/autoscaler/commands.py
+2 -4
View File
@@ -128,10 +128,8 @@ def get_or_create_head_node(config, no_restart, yes):
remote_config_file.write(json.dumps(remote_config)) remote_config_file.write(json.dumps(remote_config))
remote_config_file.flush() remote_config_file.flush()
config["file_mounts"].update({ config["file_mounts"].update({
remote_key_path: remote_key_path: config["auth"]["ssh_private_key"],
config["auth"]["ssh_private_key"], "~/ray_bootstrap_config.yaml": remote_config_file.name
"~/ray_bootstrap_config.yaml":
remote_config_file.name
}) })
if no_restart: if no_restart:
python/ray/experimental/state.py
+131 -208
View File
@@ -267,30 +267,20 @@ class GlobalState(object):
task_spec = ray.local_scheduler.task_from_string(task_spec) task_spec = ray.local_scheduler.task_from_string(task_spec)
task_spec_info = { task_spec_info = {
"DriverID": "DriverID": binary_to_hex(task_spec.driver_id().id()),
binary_to_hex(task_spec.driver_id().id()), "TaskID": binary_to_hex(task_spec.task_id().id()),
"TaskID": "ParentTaskID": binary_to_hex(task_spec.parent_task_id().id()),
binary_to_hex(task_spec.task_id().id()), "ParentCounter": task_spec.parent_counter(),
"ParentTaskID": "ActorID": binary_to_hex(task_spec.actor_id().id()),
binary_to_hex(task_spec.parent_task_id().id()), "ActorCreationID": binary_to_hex(
"ParentCounter": task_spec.actor_creation_id().id()),
task_spec.parent_counter(), "ActorCreationDummyObjectID": binary_to_hex(
"ActorID": task_spec.actor_creation_dummy_object_id().id()),
binary_to_hex(task_spec.actor_id().id()), "ActorCounter": task_spec.actor_counter(),
"ActorCreationID": "FunctionID": binary_to_hex(task_spec.function_id().id()),
binary_to_hex(task_spec.actor_creation_id().id()), "Args": task_spec.arguments(),
"ActorCreationDummyObjectID": "ReturnObjectIDs": task_spec.returns(),
binary_to_hex(task_spec.actor_creation_dummy_object_id().id()), "RequiredResources": task_spec.required_resources()
"ActorCounter":
task_spec.actor_counter(),
"FunctionID":
binary_to_hex(task_spec.function_id().id()),
"Args":
task_spec.arguments(),
"ReturnObjectIDs":
task_spec.returns(),
"RequiredResources":
task_spec.required_resources()
} }
execution_dependencies_message = ( execution_dependencies_message = (
@@ -308,18 +298,14 @@ class GlobalState(object):
# ExecutionDependencies. However, it is currently used in monitor.py. # ExecutionDependencies. However, it is currently used in monitor.py.
return { return {
"State": "State": task_table_message.State(),
task_table_message.State(), "LocalSchedulerID": binary_to_hex(
"LocalSchedulerID": task_table_message.LocalSchedulerId()),
binary_to_hex(task_table_message.LocalSchedulerId()), "ExecutionDependenciesString": task_table_message.
"ExecutionDependenciesString": ExecutionDependencies(),
task_table_message.ExecutionDependencies(), "ExecutionDependencies": execution_dependencies,
"ExecutionDependencies": "SpillbackCount": task_table_message.SpillbackCount(),
execution_dependencies, "TaskSpec": task_spec_info
"SpillbackCount":
task_table_message.SpillbackCount(),
"TaskSpec":
task_spec_info
} }
def task_table(self, task_id=None): def task_table(self, task_id=None):
@@ -652,78 +638,49 @@ class GlobalState(object):
if breakdowns: if breakdowns:
if "get_arguments_end" in info: if "get_arguments_end" in info:
get_args_trace = { get_args_trace = {
"cat": "cat": "get_arguments",
"get_arguments", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "id": task_id,
"tid": "ts": micros_rel(info["get_arguments_start"]),
info["worker_id"], "ph": "X",
"id": "name": info["function_name"] + ":get_arguments",
task_id, "args": total_info,
"ts": "dur": micros(info["get_arguments_end"] -
micros_rel(info["get_arguments_start"]), info["get_arguments_start"]),
"ph": "cname": "rail_idle"
"X",
"name":
info["function_name"] + ":get_arguments",
"args":
total_info,
"dur":
micros(info["get_arguments_end"] -
info["get_arguments_start"]),
"cname":
"rail_idle"
} }
full_trace.append(get_args_trace) full_trace.append(get_args_trace)
if "store_outputs_end" in info: if "store_outputs_end" in info:
outputs_trace = { outputs_trace = {
"cat": "cat": "store_outputs",
"store_outputs", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "id": task_id,
"tid": "ts": micros_rel(info["store_outputs_start"]),
info["worker_id"], "ph": "X",
"id": "name": info["function_name"] + ":store_outputs",
task_id, "args": total_info,
"ts": "dur": micros(info["store_outputs_end"] -
micros_rel(info["store_outputs_start"]), info["store_outputs_start"]),
"ph": "cname": "thread_state_runnable"
"X",
"name":
info["function_name"] + ":store_outputs",
"args":
total_info,
"dur":
micros(info["store_outputs_end"] -
info["store_outputs_start"]),
"cname":
"thread_state_runnable"
} }
full_trace.append(outputs_trace) full_trace.append(outputs_trace)
if "execute_end" in info: if "execute_end" in info:
execute_trace = { execute_trace = {
"cat": "cat": "execute",
"execute", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "id": task_id,
"tid": "ts": micros_rel(info["execute_start"]),
info["worker_id"], "ph": "X",
"id": "name": info["function_name"] + ":execute",
task_id, "args": total_info,
"ts": "dur": micros(info["execute_end"] -
micros_rel(info["execute_start"]), info["execute_start"]),
"ph": "cname": "rail_animation"
"X",
"name":
info["function_name"] + ":execute",
"args":
total_info,
"dur":
micros(info["execute_end"] - info["execute_start"]),
"cname":
"rail_animation"
} }
full_trace.append(execute_trace) full_trace.append(execute_trace)
@@ -733,72 +690,53 @@ class GlobalState(object):
parent_times = self._get_times(parent_info) parent_times = self._get_times(parent_info)
parent_profile = task_info.get( parent_profile = task_info.get(
task_table[task_id]["TaskSpec"]["ParentTaskID"]) task_table[task_id]["TaskSpec"]["ParentTaskID"])
_parent_id = parent_info["worker_id"] + str(
micros(min(parent_times)))
parent = { parent = {
"cat": "cat": "submit_task",
"submit_task", "pid": "Node " + parent_worker["node_ip_address"],
"pid": "tid": parent_info["worker_id"],
"Node " + parent_worker["node_ip_address"], "ts": micros_rel(
"tid": parent_profile
parent_info["worker_id"], and parent_profile["get_arguments_start"]
"ts": or start_time),
micros_rel(parent_profile "ph": "s",
and parent_profile["get_arguments_start"] "name": "SubmitTask",
or start_time),
"ph":
"s",
"name":
"SubmitTask",
"args": {}, "args": {},
"id": (parent_info["worker_id"] + str( "id": _parent_id,
micros(min(parent_times))))
} }
full_trace.append(parent) full_trace.append(parent)
_id = info["worker_id"] + str(micros(min(parent_times)))
task_trace = { task_trace = {
"cat": "cat": "submit_task",
"submit_task", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "ts": micros_rel(info["get_arguments_start"]),
"tid": "ph": "f",
info["worker_id"], "name": "SubmitTask",
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"f",
"name":
"SubmitTask",
"args": {}, "args": {},
"id": "id": _id,
(info["worker_id"] + str(micros(min(parent_times)))), "bp": "e",
"bp": "cname": "olive"
"e",
"cname":
"olive"
} }
full_trace.append(task_trace) full_trace.append(task_trace)
task = { task = {
"cat": "cat": "task",
"task", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "id": task_id,
"tid": "ts": micros_rel(info["get_arguments_start"]),
info["worker_id"], "ph": "X",
"id": "name": info["function_name"],
task_id, "args": total_info,
"ts": "dur": micros(info["store_outputs_end"] -
micros_rel(info["get_arguments_start"]), info["get_arguments_start"]),
"ph": "cname": "thread_state_runnable"
"X",
"name":
info["function_name"],
"args":
total_info,
"dur":
micros(info["store_outputs_end"] -
info["get_arguments_start"]),
"cname":
"thread_state_runnable"
} }
full_trace.append(task) full_trace.append(task)
@@ -808,45 +746,37 @@ class GlobalState(object):
parent_times = self._get_times(parent_info) parent_times = self._get_times(parent_info)
parent_profile = task_info.get( parent_profile = task_info.get(
task_table[task_id]["TaskSpec"]["ParentTaskID"]) task_table[task_id]["TaskSpec"]["ParentTaskID"])
_parent_id = parent_info["worker_id"] + str(
micros(min(parent_times)))
parent = { parent = {
"cat": "cat": "submit_task",
"submit_task", "pid": "Node " + parent_worker["node_ip_address"],
"pid": "tid": parent_info["worker_id"],
"Node " + parent_worker["node_ip_address"], "ts": micros_rel(
"tid": parent_profile
parent_info["worker_id"], and parent_profile["get_arguments_start"]
"ts": or start_time),
micros_rel(parent_profile "ph": "s",
and parent_profile["get_arguments_start"] "name": "SubmitTask",
or start_time),
"ph":
"s",
"name":
"SubmitTask",
"args": {}, "args": {},
"id": (parent_info["worker_id"] + str( "id": _parent_id,
micros(min(parent_times))))
} }
full_trace.append(parent) full_trace.append(parent)
_id = info["worker_id"] + str(micros(min(parent_times)))
task_trace = { task_trace = {
"cat": "cat": "submit_task",
"submit_task", "pid": "Node " + worker["node_ip_address"],
"pid": "tid": info["worker_id"],
"Node " + worker["node_ip_address"], "ts": micros_rel(info["get_arguments_start"]),
"tid": "ph": "f",
info["worker_id"], "name": "SubmitTask",
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"f",
"name":
"SubmitTask",
"args": {}, "args": {},
"id": "id": _id,
(info["worker_id"] + str(micros(min(parent_times)))), "bp": "e"
"bp":
"e"
} }
full_trace.append(task_trace) full_trace.append(task_trace)
@@ -879,26 +809,19 @@ class GlobalState(object):
# duration event that it's associated with, and # duration event that it's associated with, and
# the flow event therefore always gets drawn. # the flow event therefore always gets drawn.
owner = { owner = {
"cat": "cat": "obj_dependency",
"obj_dependency",
"pid": ("Node " + "pid": ("Node " +
owner_worker["node_ip_address"]), owner_worker["node_ip_address"]),
"tid": "tid": task_info[owner_task]["worker_id"],
task_info[owner_task]["worker_id"], "ts": micros_rel(task_info[owner_task]
"ts": ["store_outputs_end"]) -
micros_rel(task_info[owner_task] 2,
["store_outputs_end"]) - 2, "ph": "s",
"ph": "name": "ObjectDependency",
"s",
"name":
"ObjectDependency",
"args": {}, "args": {},
"bp": "bp": "e",
"e", "cname": "cq_build_attempt_failed",
"cname": "id": "obj" + str(arg) + str(seen_obj[arg])
"cq_build_attempt_failed",
"id":
"obj" + str(arg) + str(seen_obj[arg])
} }
full_trace.append(owner) full_trace.append(owner)
@@ -906,8 +829,8 @@ class GlobalState(object):
"cat": "obj_dependency", "cat": "obj_dependency",
"pid": "Node " + worker["node_ip_address"], "pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"], "tid": info["worker_id"],
"ts": "ts": micros_rel(info["get_arguments_start"]) +
micros_rel(info["get_arguments_start"]) + 2, 2,
"ph": "f", "ph": "f",
"name": "ObjectDependency", "name": "ObjectDependency",
"args": {}, "args": {},
@@ -985,8 +908,8 @@ class GlobalState(object):
worker_id = binary_to_hex(worker_key[len("Workers:"):]) worker_id = binary_to_hex(worker_key[len("Workers:"):])
workers_data[worker_id] = { workers_data[worker_id] = {
"local_scheduler_socket": "local_scheduler_socket": (
(worker_info[b"local_scheduler_socket"].decode("ascii")), worker_info[b"local_scheduler_socket"].decode("ascii")),
"node_ip_address": (worker_info[b"node_ip_address"] "node_ip_address": (worker_info[b"node_ip_address"]
.decode("ascii")), .decode("ascii")),
"plasma_manager_socket": (worker_info[b"plasma_manager_socket"] "plasma_manager_socket": (worker_info[b"plasma_manager_socket"]
@@ -1012,8 +935,8 @@ class GlobalState(object):
actor_info[binary_to_hex(actor_id)] = { actor_info[binary_to_hex(actor_id)] = {
"class_id": binary_to_hex(info[b"class_id"]), "class_id": binary_to_hex(info[b"class_id"]),
"driver_id": binary_to_hex(info[b"driver_id"]), "driver_id": binary_to_hex(info[b"driver_id"]),
"local_scheduler_id": "local_scheduler_id": binary_to_hex(
binary_to_hex(info[b"local_scheduler_id"]), info[b"local_scheduler_id"]),
"num_gpus": int(info[b"num_gpus"]), "num_gpus": int(info[b"num_gpus"]),
"removed": decode(info[b"removed"]) == "True" "removed": decode(info[b"removed"]) == "True"
} }
python/ray/monitor.py
+5 -3
View File
@@ -503,11 +503,13 @@ class Monitor(object):
self.cleanup_task_table() self.cleanup_task_table()
if len(self.dead_plasma_managers) > 0: if len(self.dead_plasma_managers) > 0:
self.cleanup_object_table() self.cleanup_object_table()
num_plasma_managers = len(self.live_plasma_managers) + len(
self.dead_plasma_managers)
log.debug("{} dead local schedulers, {} plasma managers total, {} " log.debug("{} dead local schedulers, {} plasma managers total, {} "
"dead plasma managers".format( "dead plasma managers".format(
len(self.dead_local_schedulers), len(self.dead_local_schedulers), num_plasma_managers,
(len(self.live_plasma_managers) + len(
self.dead_plasma_managers)),
len(self.dead_plasma_managers))) len(self.dead_plasma_managers)))
# Handle messages from the subscription channels. # Handle messages from the subscription channels.
python/ray/tune/test/trial_runner_test.py
+2 -4
View File
@@ -526,13 +526,11 @@ class VariantGeneratorTest(unittest.TestCase):
trials = generate_trials({ trials = generate_trials({
"run": "PPO", "run": "PPO",
"config": { "config": {
"x": "x": grid_search([
grid_search([
lambda spec: spec.config.y * 100, lambda spec: spec.config.y * 100,
lambda spec: spec.config.y * 200 lambda spec: spec.config.y * 200
]), ]),
"y": "y": lambda spec: 1,
lambda spec: 1,
}, },
}) })
trials = list(trials) trials = list(trials)
python/ray/tune/trainable.py
+2 -4
View File
@@ -203,10 +203,8 @@ class Trainable(object):
out = io.BytesIO() out = io.BytesIO()
with gzip.GzipFile(fileobj=out, mode="wb") as f: with gzip.GzipFile(fileobj=out, mode="wb") as f:
compressed = pickle.dumps({ compressed = pickle.dumps({
"checkpoint_name": "checkpoint_name": os.path.basename(checkpoint_prefix),
os.path.basename(checkpoint_prefix), "data": data,
"data":
data,
}) })
if len(compressed) > 10e6: # getting pretty large if len(compressed) > 10e6: # getting pretty large
print("Checkpoint size is {} bytes".format(len(compressed))) print("Checkpoint size is {} bytes".format(len(compressed)))
python/ray/worker.py
+2 -2
View File
@@ -1558,8 +1558,8 @@ def _init(address_info=None,
driver_address_info = { driver_address_info = {
"node_ip_address": node_ip_address, "node_ip_address": node_ip_address,
"redis_address": address_info["redis_address"], "redis_address": address_info["redis_address"],
"store_socket_name": "store_socket_name": (
(address_info["object_store_addresses"][0].name), address_info["object_store_addresses"][0].name),
"webui_url": address_info["webui_url"] "webui_url": address_info["webui_url"]
} }
if not use_raylet: if not use_raylet:
python/setup.py
+3 -2
View File
@@ -52,8 +52,9 @@ else:
optional_ray_files += ray_autoscaler_files optional_ray_files += ray_autoscaler_files
extras = { extras = {
"rllib": "rllib": [
["tensorflow", "pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"] "tensorflow", "pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"
]
} }
test/runtest.py
+11 -10
View File
@@ -180,14 +180,15 @@ LIST_OBJECTS = [[obj] for obj in BASE_OBJECTS]
TUPLE_OBJECTS = [(obj, ) for obj in BASE_OBJECTS] TUPLE_OBJECTS = [(obj, ) for obj in BASE_OBJECTS]
# The check that type(obj).__module__ != "numpy" should be unnecessary, but # The check that type(obj).__module__ != "numpy" should be unnecessary, but
# otherwise this seems to fail on Mac OS X on Travis. # otherwise this seems to fail on Mac OS X on Travis.
DICT_OBJECTS = ([{ DICT_OBJECTS = (
obj: obj [{
} for obj in PRIMITIVE_OBJECTS if ( obj: obj
obj.__hash__ is not None and type(obj).__module__ != "numpy")] + [{ } for obj in PRIMITIVE_OBJECTS
0: obj if (obj.__hash__ is not None and type(obj).__module__ != "numpy")] + [{
} for obj in BASE_OBJECTS] + [{ 0: obj
Foo(123): Foo(456) } for obj in BASE_OBJECTS] + [{
}]) Foo(123): Foo(456)
}])
RAY_TEST_OBJECTS = BASE_OBJECTS + LIST_OBJECTS + TUPLE_OBJECTS + DICT_OBJECTS RAY_TEST_OBJECTS = BASE_OBJECTS + LIST_OBJECTS + TUPLE_OBJECTS + DICT_OBJECTS
@@ -720,8 +721,8 @@ class APITest(unittest.TestCase):
assert ray.get([id1, id2, id3]) == [0, 1, 2] assert ray.get([id1, id2, id3]) == [0, 1, 2]
assert ray.get( assert ray.get(
g._submit( g._submit(
args=[], num_cpus=1, num_gpus=1, resources={"Custom": args=[], num_cpus=1, num_gpus=1,
1})) == [0] resources={"Custom": 1})) == [0]
infeasible_id = g._submit(args=[], resources={"NonexistentCustom": 1}) infeasible_id = g._submit(args=[], resources={"NonexistentCustom": 1})
ready_ids, remaining_ids = ray.wait([infeasible_id], timeout=50) ready_ids, remaining_ids = ray.wait([infeasible_id], timeout=50)
assert len(ready_ids) == 0 assert len(ready_ids) == 0