[GCS]Use new getting all available resources interface instead of pub-sub … (#10914)

* Use new all available resources getting interface instead of pub-sub in state.py

* add missing server handler and test cases, fix comments

* add fine grained test assert

* per comments

* involve new added function _available_resources_per_node

* change  ClientID to NodeID

* fix compile

* fix client id and lint

* robust tests check

* robust tests

python/ray/tests/test_advanced_3.py

@@ -695,12 +695,16 @@ def test_accelerator_type_api(shutdown_only):
 
     @ray.remote(accelerator_type=v100)
     def decorated_func(quantity):
-        return ray.available_resources()[resource_name] < quantity
+        wait_for_condition(
+            lambda: ray.available_resources()[resource_name] < quantity)
+        return True
 
     assert ray.get(decorated_func.remote(quantity))
 
     def via_options_func(quantity):
-        return ray.available_resources()[resource_name] < quantity
+        wait_for_condition(
+            lambda: ray.available_resources()[resource_name] < quantity)
+        return True
 
     assert ray.get(
         ray.remote(via_options_func).options(
@@ -725,13 +729,15 @@ def test_accelerator_type_api(shutdown_only):
     # Avoid a race condition where the actor hasn't been initialized and
     # claimed the resources yet.
     ray.get(decorated_actor.initialized.remote())
-    assert ray.available_resources()[resource_name] < quantity
+    wait_for_condition(
+        lambda: ray.available_resources()[resource_name] < quantity)
 
     quantity = ray.available_resources()[resource_name]
     with_options = ray.remote(ActorWithOptions).options(
         accelerator_type=v100).remote()
     ray.get(with_options.initialized.remote())
-    assert ray.available_resources()[resource_name] < quantity
+    wait_for_condition(
+        lambda: ray.available_resources()[resource_name] < quantity)
 
 
 def test_detect_docker_cpus():