From 4ecd29ea2b4988dab3fc2395af4b04e0b864537c Mon Sep 17 00:00:00 2001 From: Xianyang Liu Date: Fri, 22 Jan 2021 12:10:01 +0800 Subject: [PATCH] [dashboard] Fixes dashboard issues when environments have set http_proxy (#12598) * fixes ray start with http_proxy * format * fixes * fixes * increase timeout * address comments --- dashboard/agent.py | 3 +- dashboard/head.py | 4 +- .../modules/logical_view/logical_view_head.py | 4 +- dashboard/modules/reporter/reporter_head.py | 4 +- .../stats_collector/stats_collector_head.py | 3 +- dashboard/tests/conftest.py | 57 +++++++++++++------ dashboard/tests/test_dashboard.py | 33 +++++++++++ dashboard/utils.py | 21 +++---- 8 files changed, 97 insertions(+), 32 deletions(-) diff --git a/dashboard/agent.py b/dashboard/agent.py index f34024e54..7bf5e1551 100644 --- a/dashboard/agent.py +++ b/dashboard/agent.py @@ -75,8 +75,9 @@ class DashboardAgent(object): logger.info("Dashboard agent grpc address: %s:%s", self.ip, self.grpc_port) self.aioredis_client = None + options = (("grpc.enable_http_proxy", 0), ) self.aiogrpc_raylet_channel = aiogrpc.insecure_channel( - f"{self.ip}:{self.node_manager_port}") + f"{self.ip}:{self.node_manager_port}", options=options) self.http_session = None def _load_modules(self): diff --git a/dashboard/head.py b/dashboard/head.py index e8e911913..f1ef75ef4 100644 --- a/dashboard/head.py +++ b/dashboard/head.py @@ -159,7 +159,9 @@ class DashboardHead: if not gcs_address: raise Exception("GCS address not found.") logger.info("Connect to GCS at %s", gcs_address) - channel = aiogrpc.insecure_channel(gcs_address) + options = (("grpc.enable_http_proxy", 0), ) + channel = aiogrpc.insecure_channel( + gcs_address, options=options) except Exception as ex: logger.error("Connect to GCS failed: %s, retry...", ex) await asyncio.sleep( diff --git a/dashboard/modules/logical_view/logical_view_head.py b/dashboard/modules/logical_view/logical_view_head.py index cf29db637..6b8e0bae1 100644 --- a/dashboard/modules/logical_view/logical_view_head.py +++ b/dashboard/modules/logical_view/logical_view_head.py @@ -46,7 +46,9 @@ class LogicalViewHead(dashboard_utils.DashboardHeadModule): except KeyError: return rest_response(success=False, message="Bad Request") try: - channel = aiogrpc.insecure_channel(f"{ip_address}:{port}") + options = (("grpc.enable_http_proxy", 0), ) + channel = aiogrpc.insecure_channel( + f"{ip_address}:{port}", options=options) stub = core_worker_pb2_grpc.CoreWorkerServiceStub(channel) await stub.KillActor( diff --git a/dashboard/modules/reporter/reporter_head.py b/dashboard/modules/reporter/reporter_head.py index 8faef274d..2d84c6b65 100644 --- a/dashboard/modules/reporter/reporter_head.py +++ b/dashboard/modules/reporter/reporter_head.py @@ -38,7 +38,9 @@ class ReportHead(dashboard_utils.DashboardHeadModule): if change.new: node_id, ports = change.new ip = DataSource.node_id_to_ip[node_id] - channel = aiogrpc.insecure_channel(f"{ip}:{ports[1]}") + options = (("grpc.enable_http_proxy", 0), ) + channel = aiogrpc.insecure_channel( + f"{ip}:{ports[1]}", options=options) stub = reporter_pb2_grpc.ReporterServiceStub(channel) self._stubs[ip] = stub diff --git a/dashboard/modules/stats_collector/stats_collector_head.py b/dashboard/modules/stats_collector/stats_collector_head.py index ae75864e5..aa37e2e6e 100644 --- a/dashboard/modules/stats_collector/stats_collector_head.py +++ b/dashboard/modules/stats_collector/stats_collector_head.py @@ -71,7 +71,8 @@ class StatsCollector(dashboard_utils.DashboardHeadModule): node_id, node_info = change.new address = "{}:{}".format(node_info["nodeManagerAddress"], int(node_info["nodeManagerPort"])) - channel = aiogrpc.insecure_channel(address) + options = (("grpc.enable_http_proxy", 0), ) + channel = aiogrpc.insecure_channel(address, options=options) stub = node_manager_pb2_grpc.NodeManagerServiceStub(channel) self._stubs[node_id] = stub diff --git a/dashboard/tests/conftest.py b/dashboard/tests/conftest.py index cb49e8bfc..ec893fbef 100644 --- a/dashboard/tests/conftest.py +++ b/dashboard/tests/conftest.py @@ -1,17 +1,40 @@ -import os -import pytest -from ray.tests.conftest import * # noqa - - -@pytest.fixture -def enable_test_module(): - os.environ["RAY_DASHBOARD_MODULE_TEST"] = "true" - yield - os.environ.pop("RAY_DASHBOARD_MODULE_TEST", None) - - -@pytest.fixture -def disable_aiohttp_cache(): - os.environ["RAY_DASHBOARD_NO_CACHE"] = "true" - yield - os.environ.pop("RAY_DASHBOARD_NO_CACHE", None) +import os +import pytest +from ray.tests.conftest import * # noqa + + +@pytest.fixture +def enable_test_module(): + os.environ["RAY_DASHBOARD_MODULE_TEST"] = "true" + yield + os.environ.pop("RAY_DASHBOARD_MODULE_TEST", None) + + +@pytest.fixture +def disable_aiohttp_cache(): + os.environ["RAY_DASHBOARD_NO_CACHE"] = "true" + yield + os.environ.pop("RAY_DASHBOARD_NO_CACHE", None) + + +@pytest.fixture +def set_http_proxy(): + http_proxy = os.environ.get("http_proxy", None) + https_proxy = os.environ.get("https_proxy", None) + + # set http proxy + os.environ["http_proxy"] = "www.example.com:990" + os.environ["https_proxy"] = "www.example.com:990" + + yield + + # reset http proxy + if http_proxy: + os.environ["http_proxy"] = http_proxy + else: + del os.environ["http_proxy"] + + if https_proxy: + os.environ["https_proxy"] = https_proxy + else: + del os.environ["https_proxy"] diff --git a/dashboard/tests/test_dashboard.py b/dashboard/tests/test_dashboard.py index 1acc94a16..529e39461 100644 --- a/dashboard/tests/test_dashboard.py +++ b/dashboard/tests/test_dashboard.py @@ -571,5 +571,38 @@ def test_immutable_types(): print(d3[1]) +def test_http_proxy(enable_test_module, set_http_proxy, shutdown_only): + address_info = ray.init(num_cpus=1, include_dashboard=True) + assert (wait_until_server_available(address_info["webui_url"]) is True) + + webui_url = address_info["webui_url"] + webui_url = format_web_url(webui_url) + + timeout_seconds = 10 + start_time = time.time() + while True: + time.sleep(1) + try: + response = requests.get( + webui_url + "/test/dump", + proxies={ + "http": None, + "https": None + }) + response.raise_for_status() + try: + response.json() + assert response.ok + except Exception as ex: + logger.info("failed response: %s", response.text) + raise ex + break + except (AssertionError, requests.exceptions.ConnectionError) as e: + logger.info("Retry because of %s", e) + finally: + if time.time() > start_time + timeout_seconds: + raise Exception("Timed out while testing.") + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/dashboard/utils.py b/dashboard/utils.py index e1379eea8..5c347ed32 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -1,34 +1,35 @@ import abc -import os -import socket -import time import asyncio import collections -import json import datetime import functools import importlib import inspect +import json import logging +import os import pkgutil +import socket import traceback -from base64 import b64decode from abc import ABCMeta, abstractmethod -from collections.abc import MutableMapping, Mapping, Sequence +from base64 import b64decode from collections import namedtuple +from collections.abc import MutableMapping, Mapping, Sequence from typing import Any -import aioredis +import aiohttp.signals import aiohttp.web -import ray.new_dashboard.consts as dashboard_consts +import aioredis +import time from aiohttp import hdrs from aiohttp.frozenlist import FrozenList from aiohttp.typedefs import PathLike from aiohttp.web import RouteDef -import aiohttp.signals from google.protobuf.json_format import MessageToDict -from ray.utils import binary_to_hex + +import ray.new_dashboard.consts as dashboard_consts from ray.ray_constants import env_bool +from ray.utils import binary_to_hex try: create_task = asyncio.create_task