From d632b0f0f7b9430edca0f4a69882e298f39aa0bd Mon Sep 17 00:00:00 2001 From: Raed Shabbir Date: Mon, 4 Jan 2021 14:04:59 -0500 Subject: [PATCH] [Serve] Bug in Serve node memory-related resources calculation #11198 (#13061) --- python/ray/serve/utils.py | 13 ++++++++++++- python/ray/setup-dev.py | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/ray/serve/utils.py b/python/ray/serve/utils.py index 550ccbb45..4d41880fa 100644 --- a/python/ray/serve/utils.py +++ b/python/ray/serve/utils.py @@ -19,6 +19,7 @@ import pydantic import ray from ray.serve.constants import HTTP_PROXY_TIMEOUT +from ray.ray_constants import MEMORY_RESOURCE_UNIT_BYTES ACTOR_FAILURE_RETRY_TIMEOUT_S = 60 @@ -295,8 +296,18 @@ def try_schedule_resources_on_nodes( for node_id, node_resource in ray_resource.items(): # Check if we can schedule on this node feasible = True + for key, count in resource_dict.items(): - if node_resource.get(key, 0) - count < 0: + # Fix legacy behaviour in all memory objects + if "memory" in key: + memory_resource = node_resource.get(key, 0) + if memory_resource > 0: + # Convert from chunks to bytes + memory_resource *= MEMORY_RESOURCE_UNIT_BYTES + if memory_resource - count < 0: + feasible = False + + elif node_resource.get(key, 0) - count < 0: feasible = False # If we can, schedule it on this node diff --git a/python/ray/setup-dev.py b/python/ray/setup-dev.py index 963b2b977..d1e761183 100755 --- a/python/ray/setup-dev.py +++ b/python/ray/setup-dev.py @@ -46,6 +46,9 @@ def do_link(package, force=False, local_path=None): print("You don't have write permission " f"to {package_home}, using sudo:") sudo = ["sudo"] + print( + f"Creating symbolic link from \n {local_home} to \n {package_home}" + ) subprocess.check_call(sudo + ["rm", "-rf", package_home]) subprocess.check_call(sudo + ["ln", "-s", local_home, package_home]) @@ -67,6 +70,7 @@ if __name__ == "__main__": do_link("tests", force=args.yes) do_link("experimental", force=args.yes) do_link("util", force=args.yes) + do_link("serve", force=args.yes) # Link package's `new_dashboard` directly to local (repo's) dashboard. # The repo's `new_dashboard` is a file, soft-linking to which will not work # on Mac.