wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 17:34:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

YAPF, take 3 (#2098)

Browse Source 
* Use pep8 style

The original style file is actually just pep8 style, but with everything
spelled out. It's easier to use the `based_on_style` feature. Any overrides are
clearer that way.

* Improve yapf script

1. Do formatting in parallel
2. Lint RLlib
3. Use .style.yapf file

* Pull out expressions into variables

* Don't format rllib

* Don't allow splits in dicts

* Apply yapf

* Disallow single line if-statements

* Use arithmetic comparison

* Simplify checking for changed files

* Pull out expr into var
This commit is contained in:
Alok Singh
2018-05-19 16:07:28 -07:00
committed by Richard Liaw
parent 8e0962bb9c
commit 9a8f29e571
14 changed files with 218 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/actor.py
+14 -26
View File
@@ -857,32 +857,20 @@ class ActorHandle(object):
A dictionary of the information needed to reconstruct the object.
"""
state = {
"actor_id":
self._ray_actor_id.id(),
"class_name":
self._ray_class_name,
"actor_forks":
self._ray_actor_forks,
"actor_cursor":
self._ray_actor_cursor.id(),
"actor_counter":
0, # Reset the actor counter.
"actor_method_names":
self._ray_actor_method_names,
"method_signatures":
self._ray_method_signatures,
"method_num_return_vals":
self._ray_method_num_return_vals,
"actor_creation_dummy_object_id":
self._ray_actor_creation_dummy_object_id.id(),
"actor_method_cpus":
self._ray_actor_method_cpus,
"actor_driver_id":
self._ray_actor_driver_id.id(),
"previous_actor_handle_id":
self._ray_actor_handle_id.id(),
"ray_forking":
ray_forking
"actor_id": self._ray_actor_id.id(),
"class_name": self._ray_class_name,
"actor_forks": self._ray_actor_forks,
"actor_cursor": self._ray_actor_cursor.id(),
"actor_counter": 0, # Reset the actor counter.
"actor_method_names": self._ray_actor_method_names,
"method_signatures": self._ray_method_signatures,
"method_num_return_vals": self._ray_method_num_return_vals,
"actor_creation_dummy_object_id": self.
_ray_actor_creation_dummy_object_id.id(),
"actor_method_cpus": self._ray_actor_method_cpus,
"actor_driver_id": self._ray_actor_driver_id.id(),
"previous_actor_handle_id": self._ray_actor_handle_id.id(),
"ray_forking": ray_forking
}
if ray_forking:
python/ray/autoscaler/autoscaler.py
+4 -8
View File
@@ -182,19 +182,15 @@ class LoadMetrics(object):
nodes_used += max_frac
idle_times = [now - t for t in self.last_used_time_by_ip.values()]
return {
"ResourceUsage":
", ".join([
"ResourceUsage": ", ".join([
"{}/{} {}".format(
round(resources_used[rid], 2),
round(resources_total[rid], 2), rid)
for rid in sorted(resources_used)
]),
"NumNodesConnected":
len(self.static_resources_by_ip),
"NumNodesUsed":
round(nodes_used, 2),
"NodeIdleSeconds":
"Min={} Mean={} Max={}".format(
"NumNodesConnected": len(self.static_resources_by_ip),
"NumNodesUsed": round(nodes_used, 2),
"NodeIdleSeconds": "Min={} Mean={} Max={}".format(
int(np.min(idle_times)) if idle_times else -1,
int(np.mean(idle_times)) if idle_times else -1,
int(np.max(idle_times)) if idle_times else -1),
python/ray/autoscaler/aws/config.py
+15 -16
View File
@@ -209,22 +209,21 @@ def _configure_security_group(config):
assert security_group, "Failed to create security group"
if not security_group.ip_permissions:
security_group.authorize_ingress(
IpPermissions=[{
"FromPort": -1,
"ToPort": -1,
"IpProtocol": "-1",
"UserIdGroupPairs": [{
"GroupId": security_group.id
}]
}, {
"FromPort": 22,
"ToPort": 22,
"IpProtocol": "TCP",
"IpRanges": [{
"CidrIp": "0.0.0.0/0"
}]
}])
security_group.authorize_ingress(IpPermissions=[{
"FromPort": -1,
"ToPort": -1,
"IpProtocol": "-1",
"UserIdGroupPairs": [{
"GroupId": security_group.id
}]
}, {
"FromPort": 22,
"ToPort": 22,
"IpProtocol": "TCP",
"IpRanges": [{
"CidrIp": "0.0.0.0/0"
}]
}])
if "SecurityGroupIds" not in config["head_node"]:
print("SecurityGroupIds not specified for head node, using {}".format(
python/ray/autoscaler/aws/node_provider.py
+3 -6
View File
@@ -101,12 +101,9 @@ class AWSNodeProvider(NodeProvider):
"Value": v,
})
conf.update({
"MinCount":
1,
"MaxCount":
count,
"TagSpecifications":
conf.get("TagSpecifications", []) + [{
"MinCount": 1,
"MaxCount": count,
"TagSpecifications": conf.get("TagSpecifications", []) + [{
"ResourceType": "instance",
"Tags": tag_pairs,
}]
python/ray/autoscaler/commands.py
+2 -4
View File
@@ -128,10 +128,8 @@ def get_or_create_head_node(config, no_restart, yes):
remote_config_file.write(json.dumps(remote_config))
remote_config_file.flush()
config["file_mounts"].update({
remote_key_path:
config["auth"]["ssh_private_key"],
"~/ray_bootstrap_config.yaml":
remote_config_file.name
remote_key_path: config["auth"]["ssh_private_key"],
"~/ray_bootstrap_config.yaml": remote_config_file.name
})
if no_restart:
python/ray/experimental/state.py
+131 -208
View File
@@ -267,30 +267,20 @@ class GlobalState(object):
task_spec = ray.local_scheduler.task_from_string(task_spec)
task_spec_info = {
"DriverID":
binary_to_hex(task_spec.driver_id().id()),
"TaskID":
binary_to_hex(task_spec.task_id().id()),
"ParentTaskID":
binary_to_hex(task_spec.parent_task_id().id()),
"ParentCounter":
task_spec.parent_counter(),
"ActorID":
binary_to_hex(task_spec.actor_id().id()),
"ActorCreationID":
binary_to_hex(task_spec.actor_creation_id().id()),
"ActorCreationDummyObjectID":
binary_to_hex(task_spec.actor_creation_dummy_object_id().id()),
"ActorCounter":
task_spec.actor_counter(),
"FunctionID":
binary_to_hex(task_spec.function_id().id()),
"Args":
task_spec.arguments(),
"ReturnObjectIDs":
task_spec.returns(),
"RequiredResources":
task_spec.required_resources()
"DriverID": binary_to_hex(task_spec.driver_id().id()),
"TaskID": binary_to_hex(task_spec.task_id().id()),
"ParentTaskID": binary_to_hex(task_spec.parent_task_id().id()),
"ParentCounter": task_spec.parent_counter(),
"ActorID": binary_to_hex(task_spec.actor_id().id()),
"ActorCreationID": binary_to_hex(
task_spec.actor_creation_id().id()),
"ActorCreationDummyObjectID": binary_to_hex(
task_spec.actor_creation_dummy_object_id().id()),
"ActorCounter": task_spec.actor_counter(),
"FunctionID": binary_to_hex(task_spec.function_id().id()),
"Args": task_spec.arguments(),
"ReturnObjectIDs": task_spec.returns(),
"RequiredResources": task_spec.required_resources()
}
execution_dependencies_message = (
@@ -308,18 +298,14 @@ class GlobalState(object):
# ExecutionDependencies. However, it is currently used in monitor.py.
return {
"State":
task_table_message.State(),
"LocalSchedulerID":
binary_to_hex(task_table_message.LocalSchedulerId()),
"ExecutionDependenciesString":
task_table_message.ExecutionDependencies(),
"ExecutionDependencies":
execution_dependencies,
"SpillbackCount":
task_table_message.SpillbackCount(),
"TaskSpec":
task_spec_info
"State": task_table_message.State(),
"LocalSchedulerID": binary_to_hex(
task_table_message.LocalSchedulerId()),
"ExecutionDependenciesString": task_table_message.
ExecutionDependencies(),
"ExecutionDependencies": execution_dependencies,
"SpillbackCount": task_table_message.SpillbackCount(),
"TaskSpec": task_spec_info
}
def task_table(self, task_id=None):
@@ -652,78 +638,49 @@ class GlobalState(object):
if breakdowns:
if "get_arguments_end" in info:
get_args_trace = {
"cat":
"get_arguments",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"id":
task_id,
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"X",
"name":
info["function_name"] + ":get_arguments",
"args":
total_info,
"dur":
micros(info["get_arguments_end"] -
info["get_arguments_start"]),
"cname":
"rail_idle"
"cat": "get_arguments",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"id": task_id,
"ts": micros_rel(info["get_arguments_start"]),
"ph": "X",
"name": info["function_name"] + ":get_arguments",
"args": total_info,
"dur": micros(info["get_arguments_end"] -
info["get_arguments_start"]),
"cname": "rail_idle"
}
full_trace.append(get_args_trace)
if "store_outputs_end" in info:
outputs_trace = {
"cat":
"store_outputs",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"id":
task_id,
"ts":
micros_rel(info["store_outputs_start"]),
"ph":
"X",
"name":
info["function_name"] + ":store_outputs",
"args":
total_info,
"dur":
micros(info["store_outputs_end"] -
info["store_outputs_start"]),
"cname":
"thread_state_runnable"
"cat": "store_outputs",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"id": task_id,
"ts": micros_rel(info["store_outputs_start"]),
"ph": "X",
"name": info["function_name"] + ":store_outputs",
"args": total_info,
"dur": micros(info["store_outputs_end"] -
info["store_outputs_start"]),
"cname": "thread_state_runnable"
}
full_trace.append(outputs_trace)
if "execute_end" in info:
execute_trace = {
"cat":
"execute",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"id":
task_id,
"ts":
micros_rel(info["execute_start"]),
"ph":
"X",
"name":
info["function_name"] + ":execute",
"args":
total_info,
"dur":
micros(info["execute_end"] - info["execute_start"]),
"cname":
"rail_animation"
"cat": "execute",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"id": task_id,
"ts": micros_rel(info["execute_start"]),
"ph": "X",
"name": info["function_name"] + ":execute",
"args": total_info,
"dur": micros(info["execute_end"] -
info["execute_start"]),
"cname": "rail_animation"
}
full_trace.append(execute_trace)
@@ -733,72 +690,53 @@ class GlobalState(object):
parent_times = self._get_times(parent_info)
parent_profile = task_info.get(
task_table[task_id]["TaskSpec"]["ParentTaskID"])
_parent_id = parent_info["worker_id"] + str(
micros(min(parent_times)))
parent = {
"cat":
"submit_task",
"pid":
"Node " + parent_worker["node_ip_address"],
"tid":
parent_info["worker_id"],
"ts":
micros_rel(parent_profile
and parent_profile["get_arguments_start"]
or start_time),
"ph":
"s",
"name":
"SubmitTask",
"cat": "submit_task",
"pid": "Node " + parent_worker["node_ip_address"],
"tid": parent_info["worker_id"],
"ts": micros_rel(
parent_profile
and parent_profile["get_arguments_start"]
or start_time),
"ph": "s",
"name": "SubmitTask",
"args": {},
"id": (parent_info["worker_id"] + str(
micros(min(parent_times))))
"id": _parent_id,
}
full_trace.append(parent)
_id = info["worker_id"] + str(micros(min(parent_times)))
task_trace = {
"cat":
"submit_task",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"f",
"name":
"SubmitTask",
"cat": "submit_task",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"ts": micros_rel(info["get_arguments_start"]),
"ph": "f",
"name": "SubmitTask",
"args": {},
"id":
(info["worker_id"] + str(micros(min(parent_times)))),
"bp":
"e",
"cname":
"olive"
"id": _id,
"bp": "e",
"cname": "olive"
}
full_trace.append(task_trace)
task = {
"cat":
"task",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"id":
task_id,
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"X",
"name":
info["function_name"],
"args":
total_info,
"dur":
micros(info["store_outputs_end"] -
info["get_arguments_start"]),
"cname":
"thread_state_runnable"
"cat": "task",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"id": task_id,
"ts": micros_rel(info["get_arguments_start"]),
"ph": "X",
"name": info["function_name"],
"args": total_info,
"dur": micros(info["store_outputs_end"] -
info["get_arguments_start"]),
"cname": "thread_state_runnable"
}
full_trace.append(task)
@@ -808,45 +746,37 @@ class GlobalState(object):
parent_times = self._get_times(parent_info)
parent_profile = task_info.get(
task_table[task_id]["TaskSpec"]["ParentTaskID"])
_parent_id = parent_info["worker_id"] + str(
micros(min(parent_times)))
parent = {
"cat":
"submit_task",
"pid":
"Node " + parent_worker["node_ip_address"],
"tid":
parent_info["worker_id"],
"ts":
micros_rel(parent_profile
and parent_profile["get_arguments_start"]
or start_time),
"ph":
"s",
"name":
"SubmitTask",
"cat": "submit_task",
"pid": "Node " + parent_worker["node_ip_address"],
"tid": parent_info["worker_id"],
"ts": micros_rel(
parent_profile
and parent_profile["get_arguments_start"]
or start_time),
"ph": "s",
"name": "SubmitTask",
"args": {},
"id": (parent_info["worker_id"] + str(
micros(min(parent_times))))
"id": _parent_id,
}
full_trace.append(parent)
_id = info["worker_id"] + str(micros(min(parent_times)))
task_trace = {
"cat":
"submit_task",
"pid":
"Node " + worker["node_ip_address"],
"tid":
info["worker_id"],
"ts":
micros_rel(info["get_arguments_start"]),
"ph":
"f",
"name":
"SubmitTask",
"cat": "submit_task",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"ts": micros_rel(info["get_arguments_start"]),
"ph": "f",
"name": "SubmitTask",
"args": {},
"id":
(info["worker_id"] + str(micros(min(parent_times)))),
"bp":
"e"
"id": _id,
"bp": "e"
}
full_trace.append(task_trace)
@@ -879,26 +809,19 @@ class GlobalState(object):
# duration event that it's associated with, and
# the flow event therefore always gets drawn.
owner = {
"cat":
"obj_dependency",
"cat": "obj_dependency",
"pid": ("Node " +
owner_worker["node_ip_address"]),
"tid":
task_info[owner_task]["worker_id"],
"ts":
micros_rel(task_info[owner_task]
["store_outputs_end"]) - 2,
"ph":
"s",
"name":
"ObjectDependency",
"tid": task_info[owner_task]["worker_id"],
"ts": micros_rel(task_info[owner_task]
["store_outputs_end"]) -
2,
"ph": "s",
"name": "ObjectDependency",
"args": {},
"bp":
"e",
"cname":
"cq_build_attempt_failed",
"id":
"obj" + str(arg) + str(seen_obj[arg])
"bp": "e",
"cname": "cq_build_attempt_failed",
"id": "obj" + str(arg) + str(seen_obj[arg])
}
full_trace.append(owner)
@@ -906,8 +829,8 @@ class GlobalState(object):
"cat": "obj_dependency",
"pid": "Node " + worker["node_ip_address"],
"tid": info["worker_id"],
"ts":
micros_rel(info["get_arguments_start"]) + 2,
"ts": micros_rel(info["get_arguments_start"]) +
2,
"ph": "f",
"name": "ObjectDependency",
"args": {},
@@ -985,8 +908,8 @@ class GlobalState(object):
worker_id = binary_to_hex(worker_key[len("Workers:"):])
workers_data[worker_id] = {
"local_scheduler_socket":
(worker_info[b"local_scheduler_socket"].decode("ascii")),
"local_scheduler_socket": (
worker_info[b"local_scheduler_socket"].decode("ascii")),
"node_ip_address": (worker_info[b"node_ip_address"]
.decode("ascii")),
"plasma_manager_socket": (worker_info[b"plasma_manager_socket"]
@@ -1012,8 +935,8 @@ class GlobalState(object):
actor_info[binary_to_hex(actor_id)] = {
"class_id": binary_to_hex(info[b"class_id"]),
"driver_id": binary_to_hex(info[b"driver_id"]),
"local_scheduler_id":
binary_to_hex(info[b"local_scheduler_id"]),
"local_scheduler_id": binary_to_hex(
info[b"local_scheduler_id"]),
"num_gpus": int(info[b"num_gpus"]),
"removed": decode(info[b"removed"]) == "True"
}
python/ray/monitor.py
+5 -3
View File
@@ -503,11 +503,13 @@ class Monitor(object):
self.cleanup_task_table()
if len(self.dead_plasma_managers) > 0:
self.cleanup_object_table()
num_plasma_managers = len(self.live_plasma_managers) + len(
self.dead_plasma_managers)
log.debug("{} dead local schedulers, {} plasma managers total, {} "
"dead plasma managers".format(
len(self.dead_local_schedulers),
(len(self.live_plasma_managers) + len(
self.dead_plasma_managers)),
len(self.dead_local_schedulers), num_plasma_managers,
len(self.dead_plasma_managers)))
# Handle messages from the subscription channels.
python/ray/tune/test/trial_runner_test.py
+2 -4
View File
@@ -526,13 +526,11 @@ class VariantGeneratorTest(unittest.TestCase):
trials = generate_trials({
"run": "PPO",
"config": {
"x":
grid_search([
"x": grid_search([
lambda spec: spec.config.y * 100,
lambda spec: spec.config.y * 200
]),
"y":
lambda spec: 1,
"y": lambda spec: 1,
},
})
trials = list(trials)
python/ray/tune/trainable.py
+2 -4
View File
@@ -203,10 +203,8 @@ class Trainable(object):
out = io.BytesIO()
with gzip.GzipFile(fileobj=out, mode="wb") as f:
compressed = pickle.dumps({
"checkpoint_name":
os.path.basename(checkpoint_prefix),
"data":
data,
"checkpoint_name": os.path.basename(checkpoint_prefix),
"data": data,
})
if len(compressed) > 10e6: # getting pretty large
print("Checkpoint size is {} bytes".format(len(compressed)))
python/ray/worker.py
+2 -2
View File
@@ -1558,8 +1558,8 @@ def _init(address_info=None,
driver_address_info = {
"node_ip_address": node_ip_address,
"redis_address": address_info["redis_address"],
"store_socket_name":
(address_info["object_store_addresses"][0].name),
"store_socket_name": (
address_info["object_store_addresses"][0].name),
"webui_url": address_info["webui_url"]
}
if not use_raylet:
python/setup.py
+3 -2
View File
@@ -52,8 +52,9 @@ else:
optional_ray_files += ray_autoscaler_files
extras = {
"rllib":
["tensorflow", "pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"]
"rllib": [
"tensorflow", "pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"
]
}