diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..7edb1b7d5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,23 @@ +[flake8] +exclude = + python/ray/core/generated/ + streaming/python/generated + doc/source/conf.py + python/ray/cloudpickle/ + python/ray/thirdparty_files/ + python/build/ + python/.eggs/ +max-line-length = 79 +inline-quotes = " +ignore = + C408 + E121 + E123 + E126 + E226 + E24 + E704 + W503 + W504 + W605 +avoid-escape = no diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 46429eec4..3494aeb90 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -14,9 +14,9 @@ assignees: '' *Ray version and other system information (Python version, TensorFlow version, OS):* ### Reproduction (REQUIRED) -Please provide a script that can be run to reproduce the issue. The script should have **no external library dependencies** (i.e., use fake or mock data / environments): +Please provide a short code snippet (less than 50 lines if possible) that can be copy-pasted to reproduce the issue. The snippet should have **no external library dependencies** (i.e., use fake or mock data / environments): -If we cannot run your script, we cannot fix your issue. +If the code snippet cannot be run by itself, the issue will be closed with "needs-repro-script". - [ ] I have verified my script runs in a clean environment and reproduces the issue. - [ ] I have verified the issue also occurs with the [latest wheels](https://docs.ray.io/en/master/installation.html). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 446599849..ceefeac75 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request/Question about: For feature requests or questions, post on https://discuss.ray.io/ instead! title: '' -labels: enhancement, triage +labels: enhancement assignees: '' --- diff --git a/.gitignore b/.gitignore index 52f03c375..f7b53fa50 100644 --- a/.gitignore +++ b/.gitignore @@ -174,6 +174,8 @@ venv .*.swp *.swp tags +tags.lock +tags.temp # Emacs .#* diff --git a/.travis.yml b/.travis.yml index 64ca59411..d946fed22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -401,7 +401,7 @@ matrix: - PYTHON=3.6 - TF_VERSION=2.2.0 - TFP_VERSION=0.8 - - TORCH_VERSION=1.6 + - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED @@ -421,7 +421,7 @@ matrix: - PYTHON=3.6 - TF_VERSION=2.1.0 - TFP_VERSION=0.8 - - TORCH_VERSION=1.5 + - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED @@ -432,7 +432,7 @@ matrix: # - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 python/ray/util/sgd/... - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/... - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/... - - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/... + # - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/... # Docs: Tests and examples. - os: linux @@ -441,7 +441,7 @@ matrix: - PYTHON=3.6 - TF_VERSION=2.1.0 - TFP_VERSION=0.8 - - TORCH_VERSION=1.5 + - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_DOC_AFFECTED @@ -459,7 +459,7 @@ matrix: - INSTALL_HOROVOD=1 - TF_VERSION=2.1.0 - TFP_VERSION=0.8 - - TORCH_VERSION=1.5 + - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED diff --git a/BUILD.bazel b/BUILD.bazel index ec103528e..6ffa8df54 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -33,6 +33,7 @@ cc_library( ]), hdrs = glob([ "src/ray/rpc/*.h", + "src/ray/raylet_client/*.h", ]), copts = COPTS, strip_include_prefix = "src", @@ -55,6 +56,9 @@ cc_grpc_library( # Node manager server and client. cc_library( name = "node_manager_rpc", + srcs = glob([ + "src/ray/rpc/node_manager/*.cc", + ]), hdrs = glob([ "src/ray/rpc/node_manager/*.h", ]), @@ -809,6 +813,18 @@ cc_test( ], ) +cc_test( + name = "pull_manager_test", + srcs = [ + "src/ray/object_manager/test/pull_manager_test.cc", + ], + copts = COPTS, + deps = [ + ":raylet_lib", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "push_manager_test", srcs = [ @@ -864,11 +880,13 @@ cc_test( ) cc_test( - name = "scheduling_resources_test", - srcs = ["src/ray/common/task/scheduling_resources_test.cc"], + name = "local_placement_group_manager_test", + srcs = ["src/ray/raylet/placement_group_resource_manager_test.cc"], copts = COPTS, deps = [ + "gcs_test_util_lib", "ray_common", + "raylet_lib", "@com_google_googletest//:gtest_main", ], ) diff --git a/README.rst b/README.rst index 242f8fed7..ee025cb38 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,9 @@ .. image:: https://img.shields.io/badge/Ray-Join%20Slack-blue :target: https://forms.gle/9TSdDYUgxYs8SA9e8 +.. image:: https://img.shields.io/badge/Discuss-Ask%20Questions-blue + :target: https://discuss.ray.io/ + | @@ -155,7 +158,7 @@ RLlib Quick Start .. code-block:: bash pip install tensorflow # or tensorflow-gpu - pip install "ray[rllib]" # also recommended: ray[debug] + pip install "ray[rllib]" .. code-block:: python @@ -298,13 +301,13 @@ Getting Involved ---------------- - `Community Slack`_: Join our Slack workspace. -- `GitHub Discussions`_: For discussions about development, questions about usage, and feature requests. +- `Forum`_: For discussions about development, questions about usage, and feature requests. - `GitHub Issues`_: For reporting bugs. - `Twitter`_: Follow updates on Twitter. - `Meetup Group`_: Join our meetup group. - `StackOverflow`_: For questions about how to use Ray. -.. _`GitHub Discussions`: https://github.com/ray-project/ray/discussions +.. _`Forum`: https://discuss.ray.io/ .. _`GitHub Issues`: https://github.com/ray-project/ray/issues .. _`StackOverflow`: https://stackoverflow.com/questions/tagged/ray .. _`Meetup Group`: https://www.meetup.com/Bay-Area-Ray-Meetup/ diff --git a/build-docker.sh b/build-docker.sh index 17072693f..6989ad9c0 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -7,7 +7,7 @@ set -x GPU="" BASE_IMAGE="ubuntu:focal" -WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl" +WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl" PYTHON_VERSION="" while [[ $# -gt 0 ]] @@ -16,7 +16,7 @@ key="$1" case $key in --gpu) GPU="-gpu" - BASE_IMAGE="nvidia/cuda:10.1-cudnn7d-runtime-ubuntu18.04" + BASE_IMAGE="nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04" ;; --no-cache-build) NO_CACHE="--no-cache" diff --git a/ci/performance_tests/README.rst b/ci/performance_tests/README.rst deleted file mode 100644 index acdca55cb..000000000 --- a/ci/performance_tests/README.rst +++ /dev/null @@ -1,31 +0,0 @@ -Performance Tests -================= - -This directory contains scripts for running performance benchmarks. These -benchmarks are intended to be used by Ray developers to check if a given pull -request introduces a performance regression. - -To check if a pull request introduces a performance regression, it is necessary -to run these benchmarks on the codebase before and after the change. - -Running the Workloads ---------------------- - -To run the workload on a single machine, do the following. - -.. code-block:: bash - - python test_performance.py --num-nodes=3 - -This will start simulate a 3 node cluster on your local machine, attach to it, -and run the benchmarks. To run the benchmarks on an existing cluster, do the -following. - -.. code-block:: bash - - python test_performance.py --num-nodes=3 --address= - -The ``--num-nodes`` flag must match the number of nodes in the cluster. The -nodes in the cluster must be configured with the appropriate resource labels. In -particular, the ith node in the cluster must have a resource named ``"i"`` -with quantity ``500``. diff --git a/ci/performance_tests/test_performance.py b/ci/performance_tests/test_performance.py deleted file mode 100644 index f5095a716..000000000 --- a/ci/performance_tests/test_performance.py +++ /dev/null @@ -1,248 +0,0 @@ -import argparse -import logging -import numpy as np -import time - -import ray -from ray.tests.cluster_utils import Cluster - -logger = logging.getLogger(__name__) - -parser = argparse.ArgumentParser( - description="Parse arguments for running the performance tests.") -parser.add_argument( - "--num-nodes", - required=True, - type=int, - help="The number of nodes to simulate in the cluster.") -parser.add_argument( - "--skip-object-store-warmup", - default=False, - action="store_true", - help="True if the object store should not be warmed up. This could cause " - "the benchmarks to appear slower than usual.") -parser.add_argument( - "--address", - required=False, - type=str, - help="The address of the cluster to connect to. If this is ommitted, then " - "a cluster will be started locally (on a single machine).") - - -def start_local_cluster(num_nodes, object_store_memory): - """Start a local Ray cluster. - - The ith node in the cluster will have a resource named "i". - - Args: - num_nodes: The number of nodes to start in the cluster. - - Returns: - The cluster object. - """ - num_redis_shards = 2 - redis_max_memory = 10**8 - - cluster = Cluster() - for i in range(num_nodes): - cluster.add_node( - redis_port=6379 if i == 0 else None, - num_redis_shards=num_redis_shards if i == 0 else None, - num_cpus=8 if i == 0 else 2, - num_gpus=0, - resources={str(i): 500}, - object_store_memory=object_store_memory, - redis_max_memory=redis_max_memory) - ray.init(address=cluster.address) - - return cluster - - -def wait_for_and_check_cluster_configuration(num_nodes): - """Check that the cluster's custom resources are properly configured. - - The ith node should have a resource labeled 'i' with quantity 500. - - Args: - num_nodes: The number of nodes that we expect to be in the cluster. - - Raises: - RuntimeError: This exception is raised if the cluster is not configured - properly for this test. - """ - logger.warning("Waiting for cluster to have %s nodes.", num_nodes) - while True: - nodes = ray.nodes() - if len(nodes) == num_nodes: - break - if len(nodes) > num_nodes: - raise RuntimeError( - "The cluster has %s nodes, but it should " - "only have %s.", len(nodes), num_nodes) - if not ([set(node["Resources"].keys()) - for node in ray.nodes()] == [{str(i), "CPU"} - for i in range(num_nodes)]): - raise RuntimeError( - "The ith node in the cluster should have a " - "custom resource called 'i' with quantity " - "500. The nodes are\n%s", ray.nodes()) - if not ([[ - resource_quantity - for resource_name, resource_quantity in node["Resources"].items() - if resource_name != "CPU" - ] for node in ray.nodes()] == num_nodes * [[500.0]]): - raise RuntimeError( - "The ith node in the cluster should have a " - "custom resource called 'i' with quantity " - "500. The nodes are\n%s", ray.nodes()) - for node in ray.nodes(): - if ("0" in node["Resources"] and node["ObjectStoreSocketName"] != - ray.worker.global_worker.plasma_client.store_socket_name): - raise RuntimeError("The node that this driver is connected to " - "must have a custom resource labeled '0'.") - - -@ray.remote -def create_array(size): - return np.zeros(shape=size, dtype=np.uint8) - - -@ray.remote -def no_op(*values): - # The reason that this function takes *values is so that we can pass in - # an arbitrary number of object refs to create task dependencies. - return 1 - - -@ray.remote -class Actor(object): - def ping(self, *values): - pass - - -def warm_up_cluster(num_nodes, object_store_memory): - """Warm up the cluster. - - This will allocate enough objects in each object store to cause eviction - because the first time a driver or worker touches a region of memory in the - object store, it may be slower. - - Note that remote functions are exported lazily, so the first invocation of - a given remote function will be slower. - """ - logger.warning("Warming up the object store.") - size = object_store_memory * 2 // 5 - num_objects = 2 - while size > 0: - object_refs = [] - for i in range(num_nodes): - for _ in range(num_objects): - object_refs += [ - create_array._remote(args=[size], resources={str(i): 1}) - ] - size = size // 2 - num_objects = min(num_objects * 2, 1000) - for object_ref in object_refs: - ray.get(object_ref) - logger.warning("Finished warming up the object store.") - - # Invoke all of the remote functions once so that the definitions are - # broadcast to the workers. - ray.get(no_op.remote()) - ray.get(Actor.remote().ping.remote()) - - -def run_multiple_trials(f, num_trials): - durations = [] - for _ in range(num_trials): - start = time.time() - f() - durations.append(time.time() - start) - return durations - - -def test_tasks(num_nodes): - def one_thousand_serial_tasks_local_node(): - for _ in range(1000): - ray.get(no_op._remote(resources={"0": 1})) - - durations = run_multiple_trials(one_thousand_serial_tasks_local_node, 10) - logger.warning( - "one_thousand_serial_tasks_local_node \n" - " min: %.2gs\n" - " mean: %.2gs\n" - " std: %.2gs", np.min(durations), np.mean(durations), - np.std(durations)) - - def one_thousand_serial_tasks_remote_node(): - for _ in range(1000): - ray.get(no_op._remote(resources={"1": 1})) - - durations = run_multiple_trials(one_thousand_serial_tasks_remote_node, 10) - logger.warning( - "one_thousand_serial_tasks_remote_node \n" - " min: %.2gs\n" - " mean: %.2gs\n" - " std: %.2gs", np.min(durations), np.mean(durations), - np.std(durations)) - - def ten_thousand_parallel_tasks_local(): - ray.get([no_op._remote(resources={"0": 1}) for _ in range(10000)]) - - durations = run_multiple_trials(ten_thousand_parallel_tasks_local, 5) - logger.warning( - "ten_thousand_parallel_tasks_local \n" - " min: %.2gs\n" - " mean: %.2gs\n" - " std: %.2gs", np.min(durations), np.mean(durations), - np.std(durations)) - - def ten_thousand_parallel_tasks_load_balanced(): - ray.get([ - no_op._remote(resources={str(i % num_nodes): 1}) - for i in range(10000) - ]) - - durations = run_multiple_trials(ten_thousand_parallel_tasks_load_balanced, - 5) - logger.warning( - "ten_thousand_parallel_tasks_load_balanced \n" - " min: %.2gs\n" - " mean: %.2gs\n" - " std: %.2gs", np.min(durations), np.mean(durations), - np.std(durations)) - - -if __name__ == "__main__": - args = parser.parse_args() - num_nodes = args.num_nodes - - object_store_memory = 10**8 - - # Configure the cluster or check that it is properly configured. - - if num_nodes < 2: - raise ValueError("The --num-nodes argument must be at least 2.") - - if args.address: - ray.init(address=args.address) - wait_for_and_check_cluster_configuration(num_nodes) - logger.warning( - "Running performance benchmarks on the cluster with " - "address %s.", args.address) - else: - logger.warning( - "Running performance benchmarks on a simulated cluster " - "of %s nodes.", num_nodes) - - cluster = start_local_cluster(num_nodes, object_store_memory) - - if not args.skip_object_store_warmup: - warm_up_cluster(num_nodes, object_store_memory) - - # Run the benchmarks. - - test_tasks(num_nodes) - - # TODO(rkn): Test actors, test object transfers, test tasks with many - # dependencies. diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 2e6171662..7a6b45751 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -97,7 +97,8 @@ MYPY_FILES=( 'autoscaler/node_provider.py' 'autoscaler/sdk.py' 'autoscaler/_private/commands.py' - 'operator.py' + 'operator/operator.py' + 'operator/operator_utils.py' ) YAPF_EXCLUDES=( @@ -114,8 +115,6 @@ GIT_LS_EXCLUDES=( # TODO(barakmich): This should be cleaned up. I've at least excised the copies # of these arguments to this location, but the long-term answer is to actually # make a flake8 config file -FLAKE8_EXCLUDE="--exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files/,python/build/,python/.eggs/" -FLAKE8_IGNORES="--ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605" FLAKE8_PYX_IGNORES="--ignore=C408,E121,E123,E126,E211,E225,E226,E227,E24,E704,E999,W503,W504,W605" shellcheck_scripts() { @@ -195,10 +194,10 @@ format_all() { if [ $HAS_FLAKE8 ]; then echo "$(date)" "Flake8...." git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \ - flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_IGNORES" + flake8 --config=.flake8 git ls-files -- '*.pyx' '*.pxd' '*.pxi' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 \ - flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_PYX_IGNORES" + flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES" fi echo "$(date)" "clang-format...." @@ -237,14 +236,14 @@ format_changed() { yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}" if which flake8 >/dev/null; then git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ - flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_IGNORES" + flake8 --config=.flake8 fi fi if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' &>/dev/null; then if which flake8 >/dev/null; then git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.pyx' '*.pxd' '*.pxi' | xargs -P 5 \ - flake8 --inline-quotes '"' --no-avoid-escape "$FLAKE8_EXCLUDE" "$FLAKE8_PYX_IGNORES" + flake8 --config=.flake8 "$FLAKE8_PYX_IGNORES" fi fi diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index d11ba5867..7bc73e967 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -254,7 +254,7 @@ install_dependencies() { local torch_url="https://download.pytorch.org/whl/torch_stable.html" case "${OSTYPE}" in darwin*) pip install torch torchvision;; - *) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";; + *) pip install torch==1.7.0+cpu torchvision==0.8.1+cpu -f "${torch_url}";; esac # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others) @@ -312,12 +312,13 @@ install_dependencies() { # If CI has deemed that a different version of Tensorflow or Torch # should be installed, then upgrade/downgrade to that specific version. if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then - case "${TORCH_VERSION-1.6}" in + case "${TORCH_VERSION-1.7}" in + 1.7) TORCHVISION_VERSION=0.8.1;; 1.5) TORCHVISION_VERSION=0.6.0;; *) TORCHVISION_VERSION=0.5.0;; esac pip install --use-deprecated=legacy-resolver --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \ - torch=="${TORCH_VERSION-1.6}" torchvision=="${TORCHVISION_VERSION}" \ + torch=="${TORCH_VERSION-1.7}" torchvision=="${TORCHVISION_VERSION}" \ tensorflow=="${TF_VERSION-2.2.0}" gym fi diff --git a/cpp/src/ray/runtime/task/local_mode_task_submitter.cc b/cpp/src/ray/runtime/task/local_mode_task_submitter.cc index 3239c44a9..4d2bf1c7c 100644 --- a/cpp/src/ray/runtime/task/local_mode_task_submitter.cc +++ b/cpp/src/ray/runtime/task/local_mode_task_submitter.cc @@ -44,7 +44,7 @@ ObjectID LocalModeTaskSubmitter::Submit(InvocationSpec &invocation) { local_mode_ray_tuntime_.GetCurrentTaskId(), 0, local_mode_ray_tuntime_.GetCurrentTaskId(), address, 1, required_resources, required_placement_resources, - std::make_pair(PlacementGroupID::Nil(), -1), true); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); if (invocation.task_type == TaskType::NORMAL_TASK) { } else if (invocation.task_type == TaskType::ACTOR_CREATION_TASK) { invocation.actor_id = local_mode_ray_tuntime_.GetNextActorID(); diff --git a/cpp/src/ray/runtime/task/native_task_submitter.cc b/cpp/src/ray/runtime/task/native_task_submitter.cc index 7fb4dacc8..4e1649a51 100644 --- a/cpp/src/ray/runtime/task/native_task_submitter.cc +++ b/cpp/src/ray/runtime/task/native_task_submitter.cc @@ -27,7 +27,7 @@ ObjectID NativeTaskSubmitter::Submit(InvocationSpec &invocation) { } else { core_worker.SubmitTask(BuildRayFunction(invocation), invocation.args, TaskOptions(), &return_ids, 1, std::make_pair(PlacementGroupID::Nil(), -1), - true); + true, ""); } return return_ids[0]; } diff --git a/cpp/src/ray/runtime/task/task_executor.cc b/cpp/src/ray/runtime/task/task_executor.cc index 029d08ebc..f2b06af09 100644 --- a/cpp/src/ray/runtime/task/task_executor.cc +++ b/cpp/src/ray/runtime/task/task_executor.cc @@ -27,7 +27,7 @@ Status TaskExecutor::ExecuteTask( const std::unordered_map &required_resources, const std::vector> &args_buffer, const std::vector &arg_reference_ids, - const std::vector &return_ids, + const std::vector &return_ids, const std::string &debugger_breakpoint, std::vector> *results) { RAY_LOG(INFO) << "TaskExecutor::ExecuteTask"; RAY_CHECK(ray_function.GetLanguage() == Language::CPP); diff --git a/cpp/src/ray/runtime/task/task_executor.h b/cpp/src/ray/runtime/task/task_executor.h index 5d02f3e00..cf2d24745 100644 --- a/cpp/src/ray/runtime/task/task_executor.h +++ b/cpp/src/ray/runtime/task/task_executor.h @@ -38,7 +38,7 @@ class TaskExecutor { const std::unordered_map &required_resources, const std::vector> &args, const std::vector &arg_reference_ids, - const std::vector &return_ids, + const std::vector &return_ids, const std::string &debugger_breakpoint, std::vector> *results); virtual ~TaskExecutor(){}; diff --git a/dashboard/BUILD b/dashboard/BUILD index 4a4d0894c..70823ecef 100644 --- a/dashboard/BUILD +++ b/dashboard/BUILD @@ -13,7 +13,19 @@ py_library( py_test_run_all_subdirectory( size = "small", include = ["**/test*.py"], - exclude = ["modules/test/**"], + exclude = ["modules/test/**", "modules/stats_collector/tests/test_stats_collector.py", "tests/test_dashboard.py"], extra_srcs = [], tags = ["exclusive"], ) + +py_test( + name = "test_stats_collector", + size = "medium", + srcs = ["modules/stats_collector/tests/test_stats_collector.py"], +) + +py_test( + name = "test_dashboard", + size = "medium", + srcs = ["tests/test_dashboard.py"], +) \ No newline at end of file diff --git a/dashboard/agent.py b/dashboard/agent.py index d76c17f2d..13a6998ca 100644 --- a/dashboard/agent.py +++ b/dashboard/agent.py @@ -62,7 +62,9 @@ class DashboardAgent(object): self.object_store_name = object_store_name self.raylet_name = raylet_name self.node_id = os.environ["RAY_NODE_ID"] - assert self.node_id, "Empty node id (RAY_NODE_ID)." + self.ppid = int(os.environ["RAY_RAYLET_PID"]) + assert self.ppid > 0 + logger.info("Parent pid is %s", self.ppid) self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), )) self.grpc_port = self.server.add_insecure_port( f"[::]:{self.dashboard_agent_port}") @@ -89,17 +91,21 @@ class DashboardAgent(object): async def run(self): async def _check_parent(): - """Check if raylet is dead.""" - curr_proc = psutil.Process() - while True: - parent = curr_proc.parent() - if parent is None or parent.pid == 1: - logger.error("raylet is dead, agent will die because " - "it fate-shares with raylet.") - sys.exit(0) - await asyncio.sleep( - dashboard_consts. - DASHBOARD_AGENT_CHECK_PARENT_INTERVAL_SECONDS) + """Check if raylet is dead and fate-share if it is.""" + try: + curr_proc = psutil.Process() + while True: + parent = curr_proc.parent() + if (parent is None or parent.pid == 1 + or self.ppid != parent.pid): + logger.error("Raylet is dead, exiting.") + sys.exit(0) + await asyncio.sleep( + dashboard_consts. + DASHBOARD_AGENT_CHECK_PARENT_INTERVAL_SECONDS) + except Exception: + logger.error("Failed to check parent PID, exiting.") + sys.exit(1) check_parent_task = create_task(_check_parent()) @@ -307,4 +313,5 @@ if __name__ == "__main__": "error:\n{}".format(platform.uname()[1], traceback_str)) ray.utils.push_error_to_driver_through_redis( redis_client, ray_constants.DASHBOARD_AGENT_DIED_ERROR, message) + logger.exception(message) raise e diff --git a/dashboard/client/package-lock.json b/dashboard/client/package-lock.json index 23896de9a..8b6612942 100644 --- a/dashboard/client/package-lock.json +++ b/dashboard/client/package-lock.json @@ -1,8 +1,18177 @@ { "name": "client", "version": "0.1.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "version": "0.1.0", + "dependencies": { + "@material-ui/core": "4.11.0", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "^4.0.0-alpha.56", + "@reduxjs/toolkit": "^1.3.1", + "@types/classnames": "^2.2.10", + "@types/jest": "25.1.4", + "@types/node": "13.9.5", + "@types/react": "16.9.26", + "@types/react-dom": "16.9.5", + "@types/react-redux": "^7.1.7", + "@types/react-router-dom": "^5.1.3", + "classnames": "^2.2.6", + "react": "^16.13.1", + "react-dom": "^16.13.1", + "react-redux": "^7.2.0", + "react-router-dom": "^5.1.2", + "react-scripts": "^3.4.3", + "typeface-roboto": "0.0.75", + "typescript": "3.8.3", + "use-debounce": "^3.4.3" + }, + "devDependencies": { + "eslint-plugin-import": "^2.20.1", + "eslint-plugin-prefer-arrow": "^1.1.7", + "prettier": "^2.0.2" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.12.7.tgz", + "integrity": "sha512-YaxPMGs/XIWtYqrdEOZOCPsVWfEoriXopnsz3/i7apYPXQ3698UFhS6dVT1KN5qOsWmVgw/FOrmQgpRaZayGsw==" + }, + "node_modules/@babel/core": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.9.0.tgz", + "integrity": "sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==", + "dependencies": { + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-module-transforms": "^7.9.0", + "@babel/helpers": "^7.9.0", + "@babel/parser": "^7.9.0", + "@babel/template": "^7.8.6", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.1", + "json5": "^2.1.2", + "lodash": "^4.17.13", + "resolve": "^1.3.2", + "semver": "^5.4.1", + "source-map": "^0.5.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/@babel/generator": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.12.5.tgz", + "integrity": "sha512-m16TQQJ8hPt7E+OS/XVQg/7U184MLXtvuGbCdA7na61vha+ImkyyNM/9DDA0unYCVZn3ZOhng+qz48/KBOT96A==", + "dependencies": { + "@babel/types": "^7.12.5", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.10.4.tgz", + "integrity": "sha512-XQlqKQP4vXFB7BN8fEEerrmYvHp3fK/rBkRFz9jaJbzK0B1DSfej9Kc7ZzE8Z/OnId1jpJdNAZ3BFQjWG68rcA==", + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.10.4.tgz", + "integrity": "sha512-L0zGlFrGWZK4PbT8AszSfLTM5sDU1+Az/En9VrdT8/LmEiJt4zXt+Jve9DCAnQcbqDhCI+29y/L93mrDzddCcg==", + "dependencies": { + "@babel/helper-explode-assignable-expression": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-builder-react-jsx": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-react-jsx/-/helper-builder-react-jsx-7.10.4.tgz", + "integrity": "sha512-5nPcIZ7+KKDxT1427oBivl9V9YTal7qk0diccnh7RrcgrT/pGFOjgGw1dgryyx1GvHEpXVfoDF6Ak3rTiWh8Rg==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-builder-react-jsx-experimental": { + "version": "7.12.4", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-react-jsx-experimental/-/helper-builder-react-jsx-experimental-7.12.4.tgz", + "integrity": "sha512-AjEa0jrQqNk7eDQOo0pTfUOwQBMF+xVqrausQwT9/rTKy0g04ggFNaJpaE09IQMn9yExluigWMJcj0WC7bq+Og==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/helper-module-imports": "^7.12.1", + "@babel/types": "^7.12.1" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.12.5.tgz", + "integrity": "sha512-+qH6NrscMolUlzOYngSBMIOQpKUGPPsc61Bu5W10mg84LxZ7cmvnBHzARKbDoFxVvqqAbj6Tg6N7bSrWSPXMyw==", + "dependencies": { + "@babel/compat-data": "^7.12.5", + "@babel/helper-validator-option": "^7.12.1", + "browserslist": "^4.14.5", + "semver": "^5.5.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.12.1.tgz", + "integrity": "sha512-hkL++rWeta/OVOBTRJc9a5Azh5mt5WgZUGAKMD8JM141YsE08K//bp1unBBieO6rUKkIPyUE0USQ30jAy3Sk1w==", + "dependencies": { + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-member-expression-to-functions": "^7.12.1", + "@babel/helper-optimise-call-expression": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-split-export-declaration": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz", + "integrity": "sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "regexpu-core": "^4.7.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-define-map": { + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/helper-define-map/-/helper-define-map-7.10.5.tgz", + "integrity": "sha512-fMw4kgFB720aQFXSVaXr79pjjcW5puTCM16+rECJ/plGS+zByelE8l9nCpV1GibxTnFVmUuYG9U8wYfQHdzOEQ==", + "dependencies": { + "@babel/helper-function-name": "^7.10.4", + "@babel/types": "^7.10.5", + "lodash": "^4.17.19" + } + }, + "node_modules/@babel/helper-explode-assignable-expression": { + "version": "7.11.4", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.11.4.tgz", + "integrity": "sha512-ux9hm3zR4WV1Y3xXxXkdG/0gxF9nvI0YVmKVhvK9AfMoaQkemL3sJpXw+Xbz65azo8qJiEz2XVDUpK3KYhH3ZQ==", + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz", + "integrity": "sha512-YdaSyz1n8gY44EmN7x44zBn9zQ1Ry2Y+3GTA+3vH6Mizke1Vw0aWDM66FOYEPw8//qKkmqOckrGgTYa+6sceqQ==", + "dependencies": { + "@babel/helper-get-function-arity": "^7.10.4", + "@babel/template": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-get-function-arity": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz", + "integrity": "sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A==", + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.4.tgz", + "integrity": "sha512-wljroF5PgCk2juF69kanHVs6vrLwIPNp6DLD+Lrl3hoQ3PpPPikaDRNFA+0t81NOoMt2DL6WW/mdU8k4k6ZzuA==", + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz", + "integrity": "sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw==", + "dependencies": { + "@babel/types": "^7.12.7" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz", + "integrity": "sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA==", + "dependencies": { + "@babel/types": "^7.12.5" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.12.1.tgz", + "integrity": "sha512-QQzehgFAZ2bbISiCpmVGfiGux8YVFXQ0abBic2Envhej22DVXV9nCFaS5hIQbkyo1AdGb+gNME2TSh3hYJVV/w==", + "dependencies": { + "@babel/helper-module-imports": "^7.12.1", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-simple-access": "^7.12.1", + "@babel/helper-split-export-declaration": "^7.11.0", + "@babel/helper-validator-identifier": "^7.10.4", + "@babel/template": "^7.10.4", + "@babel/traverse": "^7.12.1", + "@babel/types": "^7.12.1", + "lodash": "^4.17.19" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.10.4.tgz", + "integrity": "sha512-n3UGKY4VXwXThEiKrgRAoVPBMqeoPgHVqiHZOanAJCG9nQUL2pLRQirUzl0ioKclHGpGqRgIOkgcIJaIWLpygg==", + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.12.1.tgz", + "integrity": "sha512-9d0KQCRM8clMPcDwo8SevNs+/9a8yWVVmaE80FGJcEP8N1qToREmWEGnBn8BUlJhYRFz6fqxeRL1sl5Ogsed7A==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/helper-wrap-function": "^7.10.4", + "@babel/types": "^7.12.1" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.12.5.tgz", + "integrity": "sha512-5YILoed0ZyIpF4gKcpZitEnXEJ9UoDRki1Ey6xz46rxOzfNMAhVIJMoune1hmPVxh40LRv1+oafz7UsWX+vyWA==", + "dependencies": { + "@babel/helper-member-expression-to-functions": "^7.12.1", + "@babel/helper-optimise-call-expression": "^7.10.4", + "@babel/traverse": "^7.12.5", + "@babel/types": "^7.12.5" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.12.1.tgz", + "integrity": "sha512-OxBp7pMrjVewSSC8fXDFrHrBcJATOOFssZwv16F3/6Xtc138GHybBfPbm9kfiqQHKhYQrlamWILwlDCeyMFEaA==", + "dependencies": { + "@babel/types": "^7.12.1" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz", + "integrity": "sha512-Mf5AUuhG1/OCChOJ/HcADmvcHM42WJockombn8ATJG3OnyiSxBK/Mm5x78BQWvmtXZKHgbjdGL2kin/HOLlZGA==", + "dependencies": { + "@babel/types": "^7.12.1" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.11.0", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.11.0.tgz", + "integrity": "sha512-74Vejvp6mHkGE+m+k5vHY93FX2cAtrw1zXrZXRlG4l410Nm9PxfEiVTn1PjDPV5SnmieiueY4AFg2xqhNFuuZg==", + "dependencies": { + "@babel/types": "^7.11.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==" + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.12.1.tgz", + "integrity": "sha512-YpJabsXlJVWP0USHjnC/AQDTLlZERbON577YUVO/wLpqyj6HAtVYnWaQaN0iUN+1/tWn3c+uKKXjRut5115Y2A==" + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.12.3", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.12.3.tgz", + "integrity": "sha512-Cvb8IuJDln3rs6tzjW3Y8UeelAOdnpB8xtQ4sme2MSZ9wOxrbThporC0y/EtE16VAtoyEfLM404Xr1e0OOp+ow==", + "dependencies": { + "@babel/helper-function-name": "^7.10.4", + "@babel/template": "^7.10.4", + "@babel/traverse": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helpers": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.10.4.tgz", + "integrity": "sha512-L2gX/XeUONeEbI78dXSrJzGdz4GQ+ZTA/aazfUsFaWjSe95kiCuOZ5HsXvkiw3iwF+mFHSRUfJU8t6YavocdXA==", + "dependencies": { + "@babel/template": "^7.10.4", + "@babel/traverse": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "dependencies": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.12.7.tgz", + "integrity": "sha512-oWR02Ubp4xTLCAqPRiNIuMVgNO5Aif/xpXtabhzW2HWUD47XJsAB4Zd/Rg30+XeQA3juXigV7hlquOTmwqLiwg==", + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-proposal-async-generator-functions": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.1.tgz", + "integrity": "sha512-d+/o30tJxFxrA1lhzJqiUcEJdI6jKlNregCv5bASeGf2Q4MXmnwH7viDo7nhx1/ohf09oaH8j1GVYG/e3Yqk6A==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-remap-async-to-generator": "^7.12.1", + "@babel/plugin-syntax-async-generators": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-decorators": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.8.3.tgz", + "integrity": "sha512-e3RvdvS4qPJVTe288DlXjwKflpfy1hr0j5dz5WpIYYeP7vQZg2WfAEIp8k5/Lwis/m5REXEteIz6rrcDtXXG7w==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-syntax-decorators": "^7.8.3" + } + }, + "node_modules/@babel/plugin-proposal-dynamic-import": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.12.1.tgz", + "integrity": "sha512-a4rhUSZFuq5W8/OO8H7BL5zspjnc1FLd9hlOxIK/f7qG4a0qsqk8uvF/ywgBA8/OmjsapjpvaEOYItfGG1qIvQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-dynamic-import": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-json-strings": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.12.1.tgz", + "integrity": "sha512-GoLDUi6U9ZLzlSda2Df++VSqDJg3CG+dR0+iWsv6XRw1rEq+zwt4DirM9yrxW6XWaTpmai1cWJLMfM8qQJf+yw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.12.1.tgz", + "integrity": "sha512-nZY0ESiaQDI1y96+jk6VxMOaL4LPo/QDHBqL+SF3/vl6dHkTwHlOI8L4ZwuRBHgakRBw5zsVylel7QPbbGuYgg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-numeric-separator": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.7.tgz", + "integrity": "sha512-8c+uy0qmnRTeukiGsjLGy6uVs/TFjJchGXUeBqlG4VWYOdJWkhhVPdQ3uHwbmalfJwv2JsV0qffXP4asRfL2SQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-object-rest-spread": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz", + "integrity": "sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.0", + "@babel/plugin-transform-parameters": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-catch-binding": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.12.1.tgz", + "integrity": "sha512-hFvIjgprh9mMw5v42sJWLI1lzU5L2sznP805zeT6rySVRA0Y18StRhDqhSxlap0oVgItRsB6WSROp4YnJTJz0g==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.7.tgz", + "integrity": "sha512-4ovylXZ0PWmwoOvhU2vhnzVNnm88/Sm9nx7V8BPgMvAzn5zDou3/Awy0EjglyubVHasJj+XCEkr/r1X3P5elCA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-skip-transparent-expression-wrappers": "^7.12.1", + "@babel/plugin-syntax-optional-chaining": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-unicode-property-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.1.tgz", + "integrity": "sha512-MYq+l+PvHuw/rKUz1at/vb6nCnQ2gmJBNaM62z0OgH7B2W1D9pvkpYtlti9bGtizNIU1K3zm4bZF9F91efVY0w==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-decorators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.10.4.tgz", + "integrity": "sha512-2NaoC6fAk2VMdhY1eerkfHV+lVYC1u8b+jmRJISqANCJlTxYy19HGdIkkQtix2UtkcPuPu+IlDgrVseZnU03bw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-flow": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.10.4.tgz", + "integrity": "sha512-yxQsX1dJixF4qEEdzVbst3SZQ58Nrooz8NV9Z9GL4byTE25BvJgl5lf0RECUf0fh28rZBb/RYTWn/eeKwCMrZQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz", + "integrity": "sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz", + "integrity": "sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.10.4.tgz", + "integrity": "sha512-oSAEz1YkBCAKr5Yiq8/BNtvSAPwkp/IyUnwZogd8p+F0RuYQQrLeRUzIQhueQTTBy/F+a40uS7OFKxnkRvmvFQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.12.1.tgz", + "integrity": "sha512-5QB50qyN44fzzz4/qxDPQMBCTHgxg3n0xRBLJUmBlLoU/sFvxVWGZF/ZUfMVDQuJUKXaBhbupxIzIfZ6Fwk/0A==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.12.1.tgz", + "integrity": "sha512-SDtqoEcarK1DFlRJ1hHRY5HvJUj5kX4qmtpMAm2QnhOlyuMC4TMdCRgW6WXpv93rZeYNeLP22y8Aq2dbcDRM1A==", + "dependencies": { + "@babel/helper-module-imports": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-remap-async-to-generator": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.1.tgz", + "integrity": "sha512-5OpxfuYnSgPalRpo8EWGPzIYf0lHBWORCkj5M0oLBwHdlux9Ri36QqGW3/LR13RSVOAoUUMzoPI/jpE4ABcHoA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.12.1.tgz", + "integrity": "sha512-zJyAC9sZdE60r1nVQHblcfCj29Dh2Y0DOvlMkcqSo0ckqjiCwNiUezUKw+RjOCwGfpLRwnAeQ2XlLpsnGkvv9w==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.1.tgz", + "integrity": "sha512-/74xkA7bVdzQTBeSUhLLJgYIcxw/dpEpCdRDiHgPJ3Mv6uC11UhjpOhl72CgqbBCmt1qtssCyB2xnJm1+PFjog==", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/helper-define-map": "^7.10.4", + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-optimise-call-expression": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-split-export-declaration": "^7.10.4", + "globals": "^11.1.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.12.1.tgz", + "integrity": "sha512-vVUOYpPWB7BkgUWPo4C44mUQHpTZXakEqFjbv8rQMg7TC6S6ZhGZ3otQcRH6u7+adSlE5i0sp63eMC/XGffrzg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.1.tgz", + "integrity": "sha512-fRMYFKuzi/rSiYb2uRLiUENJOKq4Gnl+6qOv5f8z0TZXg3llUwUhsNNwrwaT/6dUhJTzNpBr+CUvEWBtfNY1cw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.1.tgz", + "integrity": "sha512-B2pXeRKoLszfEW7J4Hg9LoFaWEbr/kzo3teWHmtFCszjRNa/b40f9mfeqZsIDLLt/FjwQ6pz/Gdlwy85xNckBA==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.12.1.tgz", + "integrity": "sha512-iRght0T0HztAb/CazveUpUQrZY+aGKKaWXMJ4uf9YJtqxSUe09j3wteztCUDRHs+SRAL7yMuFqUsLoAKKzgXjw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.12.1.tgz", + "integrity": "sha512-7tqwy2bv48q+c1EHbXK0Zx3KXd2RVQp6OC7PbwFNt/dPTAV3Lu5sWtWuAj8owr5wqtWnqHfl2/mJlUmqkChKug==", + "dependencies": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-flow-strip-types": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.9.0.tgz", + "integrity": "sha512-7Qfg0lKQhEHs93FChxVLAvhBshOPQDtJUTVHr/ZwQNRccCm4O9D79r9tVSoV8iNwjP1YgfD+e/fgHcPkN1qEQg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-syntax-flow": "^7.8.3" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.12.1.tgz", + "integrity": "sha512-Zaeq10naAsuHo7heQvyV0ptj4dlZJwZgNAtBYBnu5nNKJoW62m0zKcIEyVECrUKErkUkg6ajMy4ZfnVZciSBhg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.1.tgz", + "integrity": "sha512-JF3UgJUILoFrFMEnOJLJkRHSk6LUSXLmEFsA23aR2O5CSLUxbeUX1IZ1YQ7Sn0aXb601Ncwjx73a+FVqgcljVw==", + "dependencies": { + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.1.tgz", + "integrity": "sha512-+PxVGA+2Ag6uGgL0A5f+9rklOnnMccwEBzwYFL3EUaKuiyVnUipyXncFcfjSkbimLrODoqki1U9XxZzTvfN7IQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.1.tgz", + "integrity": "sha512-1sxePl6z9ad0gFMB9KqmYofk34flq62aqMt9NqliS/7hPEpURUCMbyHXrMPlo282iY7nAvUB1aQd5mg79UD9Jg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.12.1.tgz", + "integrity": "sha512-tDW8hMkzad5oDtzsB70HIQQRBiTKrhfgwC/KkJeGsaNFTdWhKNt/BiE8c5yj19XiGyrxpbkOfH87qkNg1YGlOQ==", + "dependencies": { + "@babel/helper-module-transforms": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.12.1.tgz", + "integrity": "sha512-dY789wq6l0uLY8py9c1B48V8mVL5gZh/+PQ5ZPrylPYsnAvnEMjqsUXkuoDVPeVK+0VyGar+D08107LzDQ6pag==", + "dependencies": { + "@babel/helper-module-transforms": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-simple-access": "^7.12.1", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.12.1.tgz", + "integrity": "sha512-Hn7cVvOavVh8yvW6fLwveFqSnd7rbQN3zJvoPNyNaQSvgfKmDBO9U1YL9+PCXGRlZD9tNdWTy5ACKqMuzyn32Q==", + "dependencies": { + "@babel/helper-hoist-variables": "^7.10.4", + "@babel/helper-module-transforms": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-validator-identifier": "^7.10.4", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.12.1.tgz", + "integrity": "sha512-aEIubCS0KHKM0zUos5fIoQm+AZUMt1ZvMpqz0/H5qAQ7vWylr9+PLYurT+Ic7ID/bKLd4q8hDovaG3Zch2uz5Q==", + "dependencies": { + "@babel/helper-module-transforms": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.1.tgz", + "integrity": "sha512-tB43uQ62RHcoDp9v2Nsf+dSM8sbNodbEicbQNA53zHz8pWUhsgHSJCGpt7daXxRydjb0KnfmB+ChXOv3oADp1Q==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.1.tgz", + "integrity": "sha512-+eW/VLcUL5L9IvJH7rT1sT0CzkdUTvPrXC2PXTn/7z7tXLBuKvezYbGdxD5WMRoyvyaujOq2fWoKl869heKjhw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.1.tgz", + "integrity": "sha512-AvypiGJH9hsquNUn+RXVcBdeE3KHPZexWRdimhuV59cSoOt5kFBmqlByorAeUlGG2CJWd0U+4ZtNKga/TB0cAw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.1.tgz", + "integrity": "sha512-xq9C5EQhdPK23ZeCdMxl8bbRnAgHFrw5EOC3KJUsSylZqdkCaFEXxGSBuTSObOpiiHHNyb82es8M1QYgfQGfNg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.1.tgz", + "integrity": "sha512-6MTCR/mZ1MQS+AwZLplX4cEySjCpnIF26ToWo942nqn8hXSm7McaHQNeGx/pt7suI1TWOWMfa/NgBhiqSnX0cQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-constant-elements": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-constant-elements/-/plugin-transform-react-constant-elements-7.10.4.tgz", + "integrity": "sha512-cYmQBW1pXrqBte1raMkAulXmi7rjg3VI6ZLg9QIic8Hq7BtYXaWuZSxsr2siOMI6SWwpxjWfnwhTUrd7JlAV7g==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "node_modules/@babel/plugin-transform-react-display-name": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.12.1.tgz", + "integrity": "sha512-cAzB+UzBIrekfYxyLlFqf/OagTvHLcVBb5vpouzkYkBclRPraiygVnafvAoipErZLI8ANv8Ecn6E/m5qPXD26w==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.12.7.tgz", + "integrity": "sha512-YFlTi6MEsclFAPIDNZYiCRbneg1MFGao9pPG9uD5htwE0vDbPaMUMeYd6itWjw7K4kro4UbdQf3ljmFl9y48dQ==", + "dependencies": { + "@babel/helper-builder-react-jsx": "^7.10.4", + "@babel/helper-builder-react-jsx-experimental": "^7.12.4", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-jsx": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-development": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.12.7.tgz", + "integrity": "sha512-Rs3ETtMtR3VLXFeYRChle5SsP/P9Jp/6dsewBQfokDSzKJThlsuFcnzLTDRALiUmTC48ej19YD9uN1mupEeEDg==", + "dependencies": { + "@babel/helper-builder-react-jsx-experimental": "^7.12.4", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-jsx": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.12.1.tgz", + "integrity": "sha512-FbpL0ieNWiiBB5tCldX17EtXgmzeEZjFrix72rQYeq9X6nUK38HCaxexzVQrZWXanxKJPKVVIU37gFjEQYkPkA==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.12.1.tgz", + "integrity": "sha512-keQ5kBfjJNRc6zZN1/nVHCd6LLIHq4aUKcVnvE/2l+ZZROSbqoiGFRtT5t3Is89XJxBQaP7NLZX2jgGHdZvvFQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.1.tgz", + "integrity": "sha512-gYrHqs5itw6i4PflFX3OdBPMQdPbF4bj2REIUxlMRUFk0/ZOAIpDFuViuxPjUL7YC8UPnf+XG7/utJvqXdPKng==", + "dependencies": { + "regenerator-transform": "^0.14.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.12.1.tgz", + "integrity": "sha512-pOnUfhyPKvZpVyBHhSBoX8vfA09b7r00Pmm1sH+29ae2hMTKVmSp4Ztsr8KBKjLjx17H0eJqaRC3bR2iThM54A==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-runtime": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz", + "integrity": "sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==", + "dependencies": { + "@babel/helper-module-imports": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "resolve": "^1.8.1", + "semver": "^5.5.1" + } + }, + "node_modules/@babel/plugin-transform-runtime/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.1.tgz", + "integrity": "sha512-GFZS3c/MhX1OusqB1MZ1ct2xRzX5ppQh2JU1h2Pnfk88HtFTM+TWQqJNfwkmxtPQtb/s1tk87oENfXJlx7rSDw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.12.1.tgz", + "integrity": "sha512-vuLp8CP0BE18zVYjsEBZ5xoCecMK6LBMMxYzJnh01rxQRvhNhH1csMMmBfNo5tGpGO+NhdSNW2mzIvBu3K1fng==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-skip-transparent-expression-wrappers": "^7.12.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.7.tgz", + "integrity": "sha512-VEiqZL5N/QvDbdjfYQBhruN0HYjSPjC4XkeqW4ny/jNtH9gcbgaqBIXYEZCNnESMAGs0/K/R7oFGMhOyu/eIxg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.12.1.tgz", + "integrity": "sha512-b4Zx3KHi+taXB1dVRBhVJtEPi9h1THCeKmae2qP0YdUHIFhVjtpqqNfxeVAa1xeHVhAy4SbHxEwx5cltAu5apw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.1.tgz", + "integrity": "sha512-EPGgpGy+O5Kg5pJFNDKuxt9RdmTgj5sgrus2XVeMp/ZIbOESadgILUbm50SNpghOh3/6yrbsH+NB5+WJTmsA7Q==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.11.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.11.0.tgz", + "integrity": "sha512-edJsNzTtvb3MaXQwj8403B7mZoGu9ElDJQZOKjGUnvilquxBA3IQoEIOvkX/1O8xfAsnHS/oQhe2w/IXrr+w0w==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.10.5", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-typescript": "^7.10.4" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.12.1.tgz", + "integrity": "sha512-SqH4ClNngh/zGwHZOOQMTD+e8FGWexILV+ePMyiDJttAWRh5dhDL8rcl5lSgU3Huiq6Zn6pWTMvdPAb21Dwdyg==", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1", + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.9.0.tgz", + "integrity": "sha512-712DeRXT6dyKAM/FMbQTV/FvRCms2hPCx+3weRjZ8iQVQWZejWWk1wwG6ViWMyqb/ouBbGOl5b6aCk0+j1NmsQ==", + "dependencies": { + "@babel/compat-data": "^7.9.0", + "@babel/helper-compilation-targets": "^7.8.7", + "@babel/helper-module-imports": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-proposal-async-generator-functions": "^7.8.3", + "@babel/plugin-proposal-dynamic-import": "^7.8.3", + "@babel/plugin-proposal-json-strings": "^7.8.3", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-proposal-numeric-separator": "^7.8.3", + "@babel/plugin-proposal-object-rest-spread": "^7.9.0", + "@babel/plugin-proposal-optional-catch-binding": "^7.8.3", + "@babel/plugin-proposal-optional-chaining": "^7.9.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.8.3", + "@babel/plugin-syntax-async-generators": "^7.8.0", + "@babel/plugin-syntax-dynamic-import": "^7.8.0", + "@babel/plugin-syntax-json-strings": "^7.8.0", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0", + "@babel/plugin-syntax-numeric-separator": "^7.8.0", + "@babel/plugin-syntax-object-rest-spread": "^7.8.0", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.0", + "@babel/plugin-syntax-optional-chaining": "^7.8.0", + "@babel/plugin-syntax-top-level-await": "^7.8.3", + "@babel/plugin-transform-arrow-functions": "^7.8.3", + "@babel/plugin-transform-async-to-generator": "^7.8.3", + "@babel/plugin-transform-block-scoped-functions": "^7.8.3", + "@babel/plugin-transform-block-scoping": "^7.8.3", + "@babel/plugin-transform-classes": "^7.9.0", + "@babel/plugin-transform-computed-properties": "^7.8.3", + "@babel/plugin-transform-destructuring": "^7.8.3", + "@babel/plugin-transform-dotall-regex": "^7.8.3", + "@babel/plugin-transform-duplicate-keys": "^7.8.3", + "@babel/plugin-transform-exponentiation-operator": "^7.8.3", + "@babel/plugin-transform-for-of": "^7.9.0", + "@babel/plugin-transform-function-name": "^7.8.3", + "@babel/plugin-transform-literals": "^7.8.3", + "@babel/plugin-transform-member-expression-literals": "^7.8.3", + "@babel/plugin-transform-modules-amd": "^7.9.0", + "@babel/plugin-transform-modules-commonjs": "^7.9.0", + "@babel/plugin-transform-modules-systemjs": "^7.9.0", + "@babel/plugin-transform-modules-umd": "^7.9.0", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.8.3", + "@babel/plugin-transform-new-target": "^7.8.3", + "@babel/plugin-transform-object-super": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.8.7", + "@babel/plugin-transform-property-literals": "^7.8.3", + "@babel/plugin-transform-regenerator": "^7.8.7", + "@babel/plugin-transform-reserved-words": "^7.8.3", + "@babel/plugin-transform-shorthand-properties": "^7.8.3", + "@babel/plugin-transform-spread": "^7.8.3", + "@babel/plugin-transform-sticky-regex": "^7.8.3", + "@babel/plugin-transform-template-literals": "^7.8.3", + "@babel/plugin-transform-typeof-symbol": "^7.8.4", + "@babel/plugin-transform-unicode-regex": "^7.8.3", + "@babel/preset-modules": "^0.1.3", + "@babel/types": "^7.9.0", + "browserslist": "^4.9.1", + "core-js-compat": "^3.6.2", + "invariant": "^2.2.2", + "levenary": "^1.1.1", + "semver": "^5.5.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.4.tgz", + "integrity": "sha512-J36NhwnfdzpmH41M1DrnkkgAqhZaqr/NBdPfQ677mLzlaXo+oDiv1deyCDtgAhz8p328otdob0Du7+xgHGZbKg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + } + }, + "node_modules/@babel/preset-react": { + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.9.1.tgz", + "integrity": "sha512-aJBYF23MPj0RNdp/4bHnAP0NVqqZRr9kl0NAOP4nJCex6OYVio59+dnQzsAWFuogdLyeaKA1hmfUIVZkY5J+TQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-transform-react-display-name": "^7.8.3", + "@babel/plugin-transform-react-jsx": "^7.9.1", + "@babel/plugin-transform-react-jsx-development": "^7.9.0", + "@babel/plugin-transform-react-jsx-self": "^7.9.0", + "@babel/plugin-transform-react-jsx-source": "^7.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-typescript": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.9.0.tgz", + "integrity": "sha512-S4cueFnGrIbvYJgwsVFKdvOmpiL0XGw9MFW9D0vgRys5g36PBhZRL8NX8Gr2akz8XRtzq6HuDXPD/1nniagNUg==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-transform-typescript": "^7.9.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.0.tgz", + "integrity": "sha512-cTIudHnzuWLS56ik4DnRnqqNf8MkdUzV4iFFI1h7Jo9xvrpQROYaAnaSd2mHLQAzzZAPfATynX5ord6YlNYNMA==", + "dependencies": { + "regenerator-runtime": "^0.13.4" + } + }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.11.2", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.11.2.tgz", + "integrity": "sha512-qh5IR+8VgFz83VBa6OkaET6uN/mJOhHONuy3m1sgF0CV6mXdPSEBdA7e1eUbVvyNtANjMbg22JUv71BaDXLY6A==", + "dependencies": { + "core-js-pure": "^3.0.0", + "regenerator-runtime": "^0.13.4" + } + }, + "node_modules/@babel/template": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.10.4.tgz", + "integrity": "sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA==", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/parser": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/traverse": { + "version": "7.12.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.12.9.tgz", + "integrity": "sha512-iX9ajqnLdoU1s1nHt36JDI9KG4k+vmI8WgjK5d+aDTwQbL2fUnzedNedssA645Ede3PM2ma1n8Q4h2ohwXgMXw==", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/generator": "^7.12.5", + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-split-export-declaration": "^7.11.0", + "@babel/parser": "^7.12.7", + "@babel/types": "^7.12.7", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.19" + } + }, + "node_modules/@babel/types": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.7.tgz", + "integrity": "sha512-MNyI92qZq6jrQkXvtIiykvl4WtoRrVV9MPn+ZfsoEENjiWcBQ3ZSHrkxnJWgWtLX3XXqX5hrSQ+X69wkmesXuQ==", + "dependencies": { + "@babel/helper-validator-identifier": "^7.10.4", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + }, + "node_modules/@cnakazawa/watch": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@cnakazawa/watch/-/watch-1.0.4.tgz", + "integrity": "sha512-v9kIhKwjeZThiWrLmj0y17CWoyddASLj9O2yvbZkbvw/N3rWOYy9zkV66ursAoVr0mV15bL8g0c4QZUE6cdDoQ==", + "dependencies": { + "exec-sh": "^0.3.2", + "minimist": "^1.2.0" + }, + "bin": { + "watch": "cli.js" + }, + "engines": { + "node": ">=0.1.95" + } + }, + "node_modules/@csstools/convert-colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@csstools/convert-colors/-/convert-colors-1.4.0.tgz", + "integrity": "sha512-5a6wqoJV/xEdbRNKVo6I4hO3VjyDq//8q2f9I6PBAvMesJHFauXDorcNCsr9RzvsZnaWi5NYCcfyqP1QeFHFbw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/@csstools/normalize.css": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@csstools/normalize.css/-/normalize.css-10.1.0.tgz", + "integrity": "sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg==" + }, + "node_modules/@emotion/hash": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", + "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==" + }, + "node_modules/@hapi/address": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", + "integrity": "sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==" + }, + "node_modules/@hapi/bourne": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@hapi/bourne/-/bourne-1.3.2.tgz", + "integrity": "sha512-1dVNHT76Uu5N3eJNTYcvxee+jzX4Z9lfciqRRHCU27ihbUcYi+iSc2iml5Ke1LXe1SyJCLA0+14Jh4tXJgOppA==" + }, + "node_modules/@hapi/hoek": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-8.5.1.tgz", + "integrity": "sha512-yN7kbciD87WzLGc5539Tn0sApjyiGHAJgKvG9W8C7O+6c7qmoQMfVs0W4bX17eqz6C78QJqqFrtgdK5EWf6Qow==" + }, + "node_modules/@hapi/joi": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/@hapi/joi/-/joi-15.1.1.tgz", + "integrity": "sha512-entf8ZMOK8sc+8YfeOlM8pCfg3b5+WZIKBfUaaJT8UsjAAPjartzxIYm3TIbjvA4u+u++KbcXD38k682nVHDAQ==", + "dependencies": { + "@hapi/address": "2.x.x", + "@hapi/bourne": "1.x.x", + "@hapi/hoek": "8.x.x", + "@hapi/topo": "3.x.x" + } + }, + "node_modules/@hapi/topo": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-3.1.6.tgz", + "integrity": "sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==", + "dependencies": { + "@hapi/hoek": "^8.3.0" + } + }, + "node_modules/@jest/console": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-24.9.0.tgz", + "integrity": "sha512-Zuj6b8TnKXi3q4ymac8EQfc3ea/uhLeCGThFqXeC8H9/raaH8ARPUTdId+XyGd03Z4In0/VjD2OYFcBF09fNLQ==", + "dependencies": { + "@jest/source-map": "^24.9.0", + "chalk": "^2.0.1", + "slash": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/console/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/console/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/console/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@jest/console/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/@jest/console/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/console/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/core": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-24.9.0.tgz", + "integrity": "sha512-Fogg3s4wlAr1VX7q+rhV9RVnUv5tD7VuWfYy1+whMiWUrvl7U3QJSJyWcDio9Lq2prqYsZaeTv2Rz24pWGkJ2A==", + "dependencies": { + "@jest/console": "^24.7.1", + "@jest/reporters": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/transform": "^24.9.0", + "@jest/types": "^24.9.0", + "ansi-escapes": "^3.0.0", + "chalk": "^2.0.1", + "exit": "^0.1.2", + "graceful-fs": "^4.1.15", + "jest-changed-files": "^24.9.0", + "jest-config": "^24.9.0", + "jest-haste-map": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-regex-util": "^24.3.0", + "jest-resolve": "^24.9.0", + "jest-resolve-dependencies": "^24.9.0", + "jest-runner": "^24.9.0", + "jest-runtime": "^24.9.0", + "jest-snapshot": "^24.9.0", + "jest-util": "^24.9.0", + "jest-validate": "^24.9.0", + "jest-watcher": "^24.9.0", + "micromatch": "^3.1.10", + "p-each-series": "^1.0.0", + "realpath-native": "^1.1.0", + "rimraf": "^2.5.4", + "slash": "^2.0.0", + "strip-ansi": "^5.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/core/node_modules/ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/core/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/core/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/core/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@jest/core/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/@jest/core/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/core/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/environment": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-24.9.0.tgz", + "integrity": "sha512-5A1QluTPhvdIPFYnO3sZC3smkNeXPVELz7ikPbhUj0bQjB07EoE9qtLrem14ZUYWdVayYbsjVwIiL4WBIMV4aQ==", + "dependencies": { + "@jest/fake-timers": "^24.9.0", + "@jest/transform": "^24.9.0", + "@jest/types": "^24.9.0", + "jest-mock": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/fake-timers": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-24.9.0.tgz", + "integrity": "sha512-eWQcNa2YSwzXWIMC5KufBh3oWRIijrQFROsIqt6v/NS9Io/gknw1jsAC9c+ih/RQX4A3O7SeWAhQeN0goKhT9A==", + "dependencies": { + "@jest/types": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-mock": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/reporters": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-24.9.0.tgz", + "integrity": "sha512-mu4X0yjaHrffOsWmVLzitKmmmWSQ3GGuefgNscUSWNiUNcEOSEQk9k3pERKEQVBb0Cnn88+UESIsZEMH3o88Gw==", + "dependencies": { + "@jest/environment": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/transform": "^24.9.0", + "@jest/types": "^24.9.0", + "chalk": "^2.0.1", + "exit": "^0.1.2", + "glob": "^7.1.2", + "istanbul-lib-coverage": "^2.0.2", + "istanbul-lib-instrument": "^3.0.1", + "istanbul-lib-report": "^2.0.4", + "istanbul-lib-source-maps": "^3.0.1", + "istanbul-reports": "^2.2.6", + "jest-haste-map": "^24.9.0", + "jest-resolve": "^24.9.0", + "jest-runtime": "^24.9.0", + "jest-util": "^24.9.0", + "jest-worker": "^24.6.0", + "node-notifier": "^5.4.2", + "slash": "^2.0.0", + "source-map": "^0.6.0", + "string-length": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/reporters/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/reporters/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/reporters/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@jest/reporters/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/@jest/reporters/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/reporters/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@jest/reporters/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/source-map": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-24.9.0.tgz", + "integrity": "sha512-/Xw7xGlsZb4MJzNDgB7PW5crou5JqWiBQaz6xyPd3ArOg2nfn/PunV8+olXbbEZzNl591o5rWKE9BRDaFAuIBg==", + "dependencies": { + "callsites": "^3.0.0", + "graceful-fs": "^4.1.15", + "source-map": "^0.6.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/source-map/node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/@jest/source-map/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@jest/test-result": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-24.9.0.tgz", + "integrity": "sha512-XEFrHbBonBJ8dGp2JmF8kP/nQI/ImPpygKHwQ/SY+es59Z3L5PI4Qb9TQQMAEeYsThG1xF0k6tmG0tIKATNiiA==", + "dependencies": { + "@jest/console": "^24.9.0", + "@jest/types": "^24.9.0", + "@types/istanbul-lib-coverage": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-24.9.0.tgz", + "integrity": "sha512-6qqsU4o0kW1dvA95qfNog8v8gkRN9ph6Lz7r96IvZpHdNipP2cBcb07J1Z45mz/VIS01OHJ3pY8T5fUY38tg4A==", + "dependencies": { + "@jest/test-result": "^24.9.0", + "jest-haste-map": "^24.9.0", + "jest-runner": "^24.9.0", + "jest-runtime": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/transform": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-24.9.0.tgz", + "integrity": "sha512-TcQUmyNRxV94S0QpMOnZl0++6RMiqpbH/ZMccFB/amku6Uwvyb1cjYX7xkp5nGNkbX4QPH/FcB6q1HBTHynLmQ==", + "dependencies": { + "@babel/core": "^7.1.0", + "@jest/types": "^24.9.0", + "babel-plugin-istanbul": "^5.1.0", + "chalk": "^2.0.1", + "convert-source-map": "^1.4.0", + "fast-json-stable-stringify": "^2.0.0", + "graceful-fs": "^4.1.15", + "jest-haste-map": "^24.9.0", + "jest-regex-util": "^24.9.0", + "jest-util": "^24.9.0", + "micromatch": "^3.1.10", + "pirates": "^4.0.1", + "realpath-native": "^1.1.0", + "slash": "^2.0.0", + "source-map": "^0.6.1", + "write-file-atomic": "2.4.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@jest/transform/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/transform/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/transform/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@jest/transform/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/@jest/transform/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/transform/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@jest/transform/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@jest/types": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", + "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", + "dependencies": { + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^1.1.1", + "@types/yargs": "^13.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@material-ui/core": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.11.0.tgz", + "integrity": "sha512-bYo9uIub8wGhZySHqLQ833zi4ZML+XCBE1XwJ8EuUVSpTWWG57Pm+YugQToJNFsEyiKFhPh8DPD0bgupz8n01g==", + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/styles": "^4.10.0", + "@material-ui/system": "^4.9.14", + "@material-ui/types": "^5.1.0", + "@material-ui/utils": "^4.10.2", + "@types/react-transition-group": "^4.2.0", + "clsx": "^1.0.4", + "hoist-non-react-statics": "^3.3.2", + "popper.js": "1.16.1-lts", + "prop-types": "^15.7.2", + "react-is": "^16.8.0", + "react-transition-group": "^4.4.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@material-ui/icons": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@material-ui/icons/-/icons-4.9.1.tgz", + "integrity": "sha512-GBitL3oBWO0hzBhvA9KxqcowRUsA0qzwKkURyC8nppnC3fw54KPKZ+d4V1Eeg/UnDRSzDaI9nGCdel/eh9AQMg==", + "dependencies": { + "@babel/runtime": "^7.4.4" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@material-ui/lab": { + "version": "4.0.0-alpha.56", + "resolved": "https://registry.npmjs.org/@material-ui/lab/-/lab-4.0.0-alpha.56.tgz", + "integrity": "sha512-xPlkK+z/6y/24ka4gVJgwPfoCF4RCh8dXb1BNE7MtF9bXEBLN/lBxNTK8VAa0qm3V2oinA6xtUIdcRh0aeRtVw==", + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "^4.10.2", + "clsx": "^1.0.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@material-ui/styles": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@material-ui/styles/-/styles-4.10.0.tgz", + "integrity": "sha512-XPwiVTpd3rlnbfrgtEJ1eJJdFCXZkHxy8TrdieaTvwxNYj42VnnCyFzxYeNW9Lhj4V1oD8YtQ6S5Gie7bZDf7Q==", + "dependencies": { + "@babel/runtime": "^7.4.4", + "@emotion/hash": "^0.8.0", + "@material-ui/types": "^5.1.0", + "@material-ui/utils": "^4.9.6", + "clsx": "^1.0.4", + "csstype": "^2.5.2", + "hoist-non-react-statics": "^3.3.2", + "jss": "^10.0.3", + "jss-plugin-camel-case": "^10.0.3", + "jss-plugin-default-unit": "^10.0.3", + "jss-plugin-global": "^10.0.3", + "jss-plugin-nested": "^10.0.3", + "jss-plugin-props-sort": "^10.0.3", + "jss-plugin-rule-value-function": "^10.0.3", + "jss-plugin-vendor-prefixer": "^10.0.3", + "prop-types": "^15.7.2" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@material-ui/system": { + "version": "4.9.14", + "resolved": "https://registry.npmjs.org/@material-ui/system/-/system-4.9.14.tgz", + "integrity": "sha512-oQbaqfSnNlEkXEziDcJDDIy8pbvwUmZXWNqlmIwDqr/ZdCK8FuV3f4nxikUh7hvClKV2gnQ9djh5CZFTHkZj3w==", + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "^4.9.6", + "csstype": "^2.5.2", + "prop-types": "^15.7.2" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@material-ui/types": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@material-ui/types/-/types-5.1.0.tgz", + "integrity": "sha512-7cqRjrY50b8QzRSYyhSpx4WRw2YuO0KKIGQEVk5J8uoz2BanawykgZGoWEqKm7pVIbzFDN0SpPcVV4IhOFkl8A==" + }, + "node_modules/@material-ui/utils": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-4.10.2.tgz", + "integrity": "sha512-eg29v74P7W5r6a4tWWDAAfZldXIzfyO1am2fIsC39hdUUHm/33k6pGOKPbgDjg/U/4ifmgAePy/1OjkKN6rFRw==", + "dependencies": { + "@babel/runtime": "^7.4.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@mrmlnc/readdir-enhanced": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", + "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==", + "dependencies": { + "call-me-maybe": "^1.0.1", + "glob-to-regexp": "^0.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz", + "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/@reduxjs/toolkit": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.3.1.tgz", + "integrity": "sha512-YIAgVs3MVR+5sGl4QcPvzzr4cWY93MFt6KLZhuLYC0ykD2kZEX2+5zmYT1O01p9qFXsNzbB6FyWMt95+ayhBUQ==", + "dependencies": { + "immer": "^6.0.1", + "redux": "^4.0.0", + "redux-thunk": "^2.3.0", + "reselect": "^4.0.0" + } + }, + "node_modules/@svgr/babel-plugin-add-jsx-attribute": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-4.2.0.tgz", + "integrity": "sha512-j7KnilGyZzYr/jhcrSYS3FGWMZVaqyCG0vzMCwzvei0coIkczuYMcniK07nI0aHJINciujjH11T72ICW5eL5Ig==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-attribute": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-attribute/-/babel-plugin-remove-jsx-attribute-4.2.0.tgz", + "integrity": "sha512-3XHLtJ+HbRCH4n28S7y/yZoEQnRpl0tvTZQsHqvaeNXPra+6vE5tbRliH3ox1yZYPCxrlqaJT/Mg+75GpDKlvQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-remove-jsx-empty-expression": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-remove-jsx-empty-expression/-/babel-plugin-remove-jsx-empty-expression-4.2.0.tgz", + "integrity": "sha512-yTr2iLdf6oEuUE9MsRdvt0NmdpMBAkgK8Bjhl6epb+eQWk6abBaX3d65UZ3E3FWaOwePyUgNyNCMVG61gGCQ7w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-replace-jsx-attribute-value": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-replace-jsx-attribute-value/-/babel-plugin-replace-jsx-attribute-value-4.2.0.tgz", + "integrity": "sha512-U9m870Kqm0ko8beHawRXLGLvSi/ZMrl89gJ5BNcT452fAjtF2p4uRzXkdzvGJJJYBgx7BmqlDjBN/eCp5AAX2w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-svg-dynamic-title": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-dynamic-title/-/babel-plugin-svg-dynamic-title-4.3.3.tgz", + "integrity": "sha512-w3Be6xUNdwgParsvxkkeZb545VhXEwjGMwExMVBIdPQJeyMQHqm9Msnb2a1teHBqUYL66qtwfhNkbj1iarCG7w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-svg-em-dimensions": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-svg-em-dimensions/-/babel-plugin-svg-em-dimensions-4.2.0.tgz", + "integrity": "sha512-C0Uy+BHolCHGOZ8Dnr1zXy/KgpBOkEUYY9kI/HseHVPeMbluaX3CijJr7D4C5uR8zrc1T64nnq/k63ydQuGt4w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-transform-react-native-svg": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-react-native-svg/-/babel-plugin-transform-react-native-svg-4.2.0.tgz", + "integrity": "sha512-7YvynOpZDpCOUoIVlaaOUU87J4Z6RdD6spYN4eUb5tfPoKGSF9OG2NuhgYnq4jSkAxcpMaXWPf1cePkzmqTPNw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-plugin-transform-svg-component": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@svgr/babel-plugin-transform-svg-component/-/babel-plugin-transform-svg-component-4.2.0.tgz", + "integrity": "sha512-hYfYuZhQPCBVotABsXKSCfel2slf/yvJY8heTVX1PCTaq/IgASq1IyxPPKJ0chWREEKewIU/JMSsIGBtK1KKxw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/babel-preset": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@svgr/babel-preset/-/babel-preset-4.3.3.tgz", + "integrity": "sha512-6PG80tdz4eAlYUN3g5GZiUjg2FMcp+Wn6rtnz5WJG9ITGEF1pmFdzq02597Hn0OmnQuCVaBYQE1OVFAnwOl+0A==", + "dependencies": { + "@svgr/babel-plugin-add-jsx-attribute": "^4.2.0", + "@svgr/babel-plugin-remove-jsx-attribute": "^4.2.0", + "@svgr/babel-plugin-remove-jsx-empty-expression": "^4.2.0", + "@svgr/babel-plugin-replace-jsx-attribute-value": "^4.2.0", + "@svgr/babel-plugin-svg-dynamic-title": "^4.3.3", + "@svgr/babel-plugin-svg-em-dimensions": "^4.2.0", + "@svgr/babel-plugin-transform-react-native-svg": "^4.2.0", + "@svgr/babel-plugin-transform-svg-component": "^4.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/core": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@svgr/core/-/core-4.3.3.tgz", + "integrity": "sha512-qNuGF1QON1626UCaZamWt5yedpgOytvLj5BQZe2j1k1B8DUG4OyugZyfEwBeXozCUwhLEpsrgPrE+eCu4fY17w==", + "dependencies": { + "@svgr/plugin-jsx": "^4.3.3", + "camelcase": "^5.3.1", + "cosmiconfig": "^5.2.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/hast-util-to-babel-ast": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@svgr/hast-util-to-babel-ast/-/hast-util-to-babel-ast-4.3.2.tgz", + "integrity": "sha512-JioXclZGhFIDL3ddn4Kiq8qEqYM2PyDKV0aYno8+IXTLuYt6TOgHUbUAAFvqtb0Xn37NwP0BTHglejFoYr8RZg==", + "dependencies": { + "@babel/types": "^7.4.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/plugin-jsx": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@svgr/plugin-jsx/-/plugin-jsx-4.3.3.tgz", + "integrity": "sha512-cLOCSpNWQnDB1/v+SUENHH7a0XY09bfuMKdq9+gYvtuwzC2rU4I0wKGFEp1i24holdQdwodCtDQdFtJiTCWc+w==", + "dependencies": { + "@babel/core": "^7.4.5", + "@svgr/babel-preset": "^4.3.3", + "@svgr/hast-util-to-babel-ast": "^4.3.2", + "svg-parser": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/plugin-svgo": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@svgr/plugin-svgo/-/plugin-svgo-4.3.1.tgz", + "integrity": "sha512-PrMtEDUWjX3Ea65JsVCwTIXuSqa3CG9px+DluF1/eo9mlDrgrtFE7NE/DjdhjJgSM9wenlVBzkzneSIUgfUI/w==", + "dependencies": { + "cosmiconfig": "^5.2.1", + "merge-deep": "^3.0.2", + "svgo": "^1.2.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@svgr/webpack": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@svgr/webpack/-/webpack-4.3.3.tgz", + "integrity": "sha512-bjnWolZ6KVsHhgyCoYRFmbd26p8XVbulCzSG53BDQqAr+JOAderYK7CuYrB3bDjHJuF6LJ7Wrr42+goLRV9qIg==", + "dependencies": { + "@babel/core": "^7.4.5", + "@babel/plugin-transform-react-constant-elements": "^7.0.0", + "@babel/preset-env": "^7.4.5", + "@babel/preset-react": "^7.0.0", + "@svgr/core": "^4.3.3", + "@svgr/plugin-jsx": "^4.3.3", + "@svgr/plugin-svgo": "^4.3.1", + "loader-utils": "^1.2.3" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@types/babel__core": { + "version": "7.1.9", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.9.tgz", + "integrity": "sha512-sY2RsIJ5rpER1u3/aQ8OFSI7qGIy8o1NEEbgb2UaJcvOtXOMpd39ko723NBpjQFg9SIX7TXtjejZVGeIMLhoOw==", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.1.tgz", + "integrity": "sha512-bBKm+2VPJcMRVwNhxKu8W+5/zT7pwNEqeokFOmbvVSqGzFneNxYcEBro9Ac7/N9tlsaPYnZLK8J1LWKkMsLAew==", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.0.2.tgz", + "integrity": "sha512-/K6zCpeW7Imzgab2bLkLEbz0+1JlFSrUMdw7KoIIu+IUdu51GWaBZpd3y1VXGVXzynvGa4DaIaxNZHiON3GXUg==", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.0.13", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.0.13.tgz", + "integrity": "sha512-i+zS7t6/s9cdQvbqKDARrcbrPvtJGlbYsMkazo03nTAK3RX9FNrLllXys22uiTGJapPOTZTQ35nHh4ISph4SLQ==", + "dependencies": { + "@babel/types": "^7.3.0" + } + }, + "node_modules/@types/classnames": { + "version": "2.2.10", + "resolved": "https://registry.npmjs.org/@types/classnames/-/classnames-2.2.10.tgz", + "integrity": "sha512-1UzDldn9GfYYEsWWnn/P4wkTlkZDH7lDb0wBMGbtIQc9zXEQq7FlKBdZUn6OBqD8sKZZ2RQO2mAjGpXiDGoRmQ==" + }, + "node_modules/@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==" + }, + "node_modules/@types/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==", + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "node_modules/@types/history": { + "version": "4.7.5", + "resolved": "https://registry.npmjs.org/@types/history/-/history-4.7.5.tgz", + "integrity": "sha512-wLD/Aq2VggCJXSjxEwrMafIP51Z+13H78nXIX0ABEuIGhmB5sNGbR113MOKo+yfw+RDo1ZU3DM6yfnnRF/+ouw==" + }, + "node_modules/@types/hoist-non-react-statics": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz", + "integrity": "sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA==", + "dependencies": { + "@types/react": "*", + "hoist-non-react-statics": "^3.3.0" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz", + "integrity": "sha512-hRJD2ahnnpLgsj6KWMYSrmXkM3rm2Dl1qkx6IOFD5FnuNPXJIG5L0dhgKXCYTRMGzU4n0wImQ/xfmRc4POUFlg==" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz", + "integrity": "sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg==", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-1.1.1.tgz", + "integrity": "sha512-UpYjBi8xefVChsCoBpKShdxTllC9pwISirfoZsUa2AAdQg/Jd2KQGtSbw+ya7GPo7x/wAPlH6JBhKhAsXUEZNA==", + "dependencies": { + "@types/istanbul-lib-coverage": "*", + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "25.1.4", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-25.1.4.tgz", + "integrity": "sha512-QDDY2uNAhCV7TMCITrxz+MRk1EizcsevzfeS6LykIlq2V1E5oO4wXG8V2ZEd9w7Snxeeagk46YbMgZ8ESHx3sw==", + "dependencies": { + "jest-diff": "^25.1.0", + "pretty-format": "^25.1.0" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", + "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==" + }, + "node_modules/@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" + }, + "node_modules/@types/node": { + "version": "13.9.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.9.5.tgz", + "integrity": "sha512-hkzMMD3xu6BrJpGVLeQ3htQQNAcOrJjX7WFmtK8zWQpz2UJf13LCFF2ALA7c9OVdvc2vQJeDdjfR35M0sBCxvw==" + }, + "node_modules/@types/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" + }, + "node_modules/@types/prop-types": { + "version": "15.7.3", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.3.tgz", + "integrity": "sha512-KfRL3PuHmqQLOG+2tGpRO26Ctg+Cq1E01D2DMriKEATHgWLfeNDmq9e29Q9WIky0dQ3NPkd1mzYH8Lm936Z9qw==" + }, + "node_modules/@types/q": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz", + "integrity": "sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug==" + }, + "node_modules/@types/react": { + "version": "16.9.26", + "resolved": "https://registry.npmjs.org/@types/react/-/react-16.9.26.tgz", + "integrity": "sha512-dGuSM+B0Pq1MKXYUMlUQWeS6Jj9IhSAUf9v8Ikaimj+YhkBcQrihWBkmyEhK/1fzkJTwZQkhZp5YhmWa2CH+Rw==", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^2.2.0" + } + }, + "node_modules/@types/react-dom": { + "version": "16.9.5", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-16.9.5.tgz", + "integrity": "sha512-BX6RQ8s9D+2/gDhxrj8OW+YD4R+8hj7FEM/OJHGNR0KipE1h1mSsf39YeyC81qafkq+N3rU3h3RFbLSwE5VqUg==", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/react-redux": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/@types/react-redux/-/react-redux-7.1.7.tgz", + "integrity": "sha512-U+WrzeFfI83+evZE2dkZ/oF/1vjIYgqrb5dGgedkqVV8HEfDFujNgWCwHL89TDuWKb47U0nTBT6PLGq4IIogWg==", + "dependencies": { + "@types/hoist-non-react-statics": "^3.3.0", + "@types/react": "*", + "hoist-non-react-statics": "^3.3.0", + "redux": "^4.0.0" + } + }, + "node_modules/@types/react-router": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/@types/react-router/-/react-router-5.1.4.tgz", + "integrity": "sha512-PZtnBuyfL07sqCJvGg3z+0+kt6fobc/xmle08jBiezLS8FrmGeiGkJnuxL/8Zgy9L83ypUhniV5atZn/L8n9MQ==", + "dependencies": { + "@types/history": "*", + "@types/react": "*" + } + }, + "node_modules/@types/react-router-dom": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@types/react-router-dom/-/react-router-dom-5.1.3.tgz", + "integrity": "sha512-pCq7AkOvjE65jkGS5fQwQhvUp4+4PVD9g39gXLZViP2UqFiFzsEpB3PKf0O6mdbKsewSK8N14/eegisa/0CwnA==", + "dependencies": { + "@types/history": "*", + "@types/react": "*", + "@types/react-router": "*" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.0.tgz", + "integrity": "sha512-/QfLHGpu+2fQOqQaXh8MG9q03bFENooTb/it4jr5kKaZlDQfWvjqWZg48AwzPVMBHlRuTRAY7hRHCEOXz5kV6w==", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/stack-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", + "integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==" + }, + "node_modules/@types/yargs": { + "version": "13.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.11.tgz", + "integrity": "sha512-NRqD6T4gktUrDi1o1wLH3EKC1o2caCr7/wR87ODcbVITQF106OM3sFN92ysZ++wqelOd1CTzatnOBRDYYG6wGQ==", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-15.0.0.tgz", + "integrity": "sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw==" + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-2.34.0.tgz", + "integrity": "sha512-4zY3Z88rEE99+CNvTbXSyovv2z9PNOVffTWD2W8QF5s2prBQtwN2zadqERcrHpcR7O/+KMI3fcTAmUUhK/iQcQ==", + "dependencies": { + "@typescript-eslint/experimental-utils": "2.34.0", + "functional-red-black-tree": "^1.0.1", + "regexpp": "^3.0.0", + "tsutils": "^3.17.1" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + } + }, + "node_modules/@typescript-eslint/experimental-utils": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-2.34.0.tgz", + "integrity": "sha512-eS6FTkq+wuMJ+sgtuNTtcqavWXqsflWcfBnlYhg/nS4aZ1leewkXGbvBhaapn1q6qf4M71bsR1tez5JTRMuqwA==", + "dependencies": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-2.34.0.tgz", + "integrity": "sha512-03ilO0ucSD0EPTw2X4PntSIRFtDPWjrVq7C3/Z3VQHRC7+13YB55rcJI3Jt+YgeHbjUdJPcPa7b23rXCBokuyA==", + "dependencies": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "2.34.0", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz", + "integrity": "sha512-OMAr+nJWKdlVM9LOqCqh3pQQPwxHAN7Du8DR6dmwCrAmxtiXQnhHJ6tBNtf+cggqfo51SG/FCwnKhXCIM7hnVg==", + "dependencies": { + "debug": "^4.1.1", + "eslint-visitor-keys": "^1.1.0", + "glob": "^7.1.6", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.8.5.tgz", + "integrity": "sha512-aJMfngIZ65+t71C3y2nBBg5FFG0Okt9m0XEgWZ7Ywgn1oMAT8cNwx00Uv1cQyHtidq0Xn94R4TAywO+LCQ+ZAQ==", + "dependencies": { + "@webassemblyjs/helper-module-context": "1.8.5", + "@webassemblyjs/helper-wasm-bytecode": "1.8.5", + "@webassemblyjs/wast-parser": "1.8.5" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.8.5.tgz", + "integrity": "sha512-9p+79WHru1oqBh9ewP9zW95E3XAo+90oth7S5Re3eQnECGq59ly1Ri5tsIipKGpiStHsUYmY3zMLqtk3gTcOtQ==" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.8.5.tgz", + "integrity": "sha512-Za/tnzsvnqdaSPOUXHyKJ2XI7PDX64kWtURyGiJJZKVEdFOsdKUCPTNEVFZq3zJ2R0G5wc2PZ5gvdTRFgm81zA==" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.8.5.tgz", + "integrity": "sha512-Ri2R8nOS0U6G49Q86goFIPNgjyl6+oE1abW1pS84BuhP1Qcr5JqMwRFT3Ah3ADDDYGEgGs1iyb1DGX+kAi/c/Q==" + }, + "node_modules/@webassemblyjs/helper-code-frame": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.8.5.tgz", + "integrity": "sha512-VQAadSubZIhNpH46IR3yWO4kZZjMxN1opDrzePLdVKAZ+DFjkGD/rf4v1jap744uPVU6yjL/smZbRIIJTOUnKQ==", + "dependencies": { + "@webassemblyjs/wast-printer": "1.8.5" + } + }, + "node_modules/@webassemblyjs/helper-fsm": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.8.5.tgz", + "integrity": "sha512-kRuX/saORcg8se/ft6Q2UbRpZwP4y7YrWsLXPbbmtepKr22i8Z4O3V5QE9DbZK908dh5Xya4Un57SDIKwB9eow==" + }, + "node_modules/@webassemblyjs/helper-module-context": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.8.5.tgz", + "integrity": "sha512-/O1B236mN7UNEU4t9X7Pj38i4VoU8CcMHyy3l2cV/kIF4U5KoHXDVqcDuOs1ltkac90IM4vZdHc52t1x8Yfs3g==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "mamacro": "^0.0.3" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.8.5.tgz", + "integrity": "sha512-Cu4YMYG3Ddl72CbmpjU/wbP6SACcOPVbHN1dI4VJNJVgFwaKf1ppeFJrwydOG3NDHxVGuCfPlLZNyEdIYlQ6QQ==" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.8.5.tgz", + "integrity": "sha512-VV083zwR+VTrIWWtgIUpqfvVdK4ff38loRmrdDBgBT8ADXYsEZ5mPQ4Nde90N3UYatHdYoDIFb7oHzMncI02tA==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-buffer": "1.8.5", + "@webassemblyjs/helper-wasm-bytecode": "1.8.5", + "@webassemblyjs/wasm-gen": "1.8.5" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.8.5.tgz", + "integrity": "sha512-aaCvQYrvKbY/n6wKHb/ylAJr27GglahUO89CcGXMItrOBqRarUMxWLJgxm9PJNuKULwN5n1csT9bYoMeZOGF3g==", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.8.5.tgz", + "integrity": "sha512-plYUuUwleLIziknvlP8VpTgO4kqNaH57Y3JnNa6DLpu/sGcP6hbVdfdX5aHAV716pQBKrfuU26BJK29qY37J7A==", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.8.5.tgz", + "integrity": "sha512-U7zgftmQriw37tfD934UNInokz6yTmn29inT2cAetAsaU9YeVCveWEwhKL1Mg4yS7q//NGdzy79nlXh3bT8Kjw==" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.8.5.tgz", + "integrity": "sha512-A41EMy8MWw5yvqj7MQzkDjU29K7UJq1VrX2vWLzfpRHt3ISftOXqrtojn7nlPsZ9Ijhp5NwuODuycSvfAO/26Q==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-buffer": "1.8.5", + "@webassemblyjs/helper-wasm-bytecode": "1.8.5", + "@webassemblyjs/helper-wasm-section": "1.8.5", + "@webassemblyjs/wasm-gen": "1.8.5", + "@webassemblyjs/wasm-opt": "1.8.5", + "@webassemblyjs/wasm-parser": "1.8.5", + "@webassemblyjs/wast-printer": "1.8.5" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.8.5.tgz", + "integrity": "sha512-BCZBT0LURC0CXDzj5FXSc2FPTsxwp3nWcqXQdOZE4U7h7i8FqtFK5Egia6f9raQLpEKT1VL7zr4r3+QX6zArWg==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-wasm-bytecode": "1.8.5", + "@webassemblyjs/ieee754": "1.8.5", + "@webassemblyjs/leb128": "1.8.5", + "@webassemblyjs/utf8": "1.8.5" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.8.5.tgz", + "integrity": "sha512-HKo2mO/Uh9A6ojzu7cjslGaHaUU14LdLbGEKqTR7PBKwT6LdPtLLh9fPY33rmr5wcOMrsWDbbdCHq4hQUdd37Q==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-buffer": "1.8.5", + "@webassemblyjs/wasm-gen": "1.8.5", + "@webassemblyjs/wasm-parser": "1.8.5" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.8.5.tgz", + "integrity": "sha512-pi0SYE9T6tfcMkthwcgCpL0cM9nRYr6/6fjgDtL6q/ZqKHdMWvxitRi5JcZ7RI4SNJJYnYNaWy5UUrHQy998lw==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-api-error": "1.8.5", + "@webassemblyjs/helper-wasm-bytecode": "1.8.5", + "@webassemblyjs/ieee754": "1.8.5", + "@webassemblyjs/leb128": "1.8.5", + "@webassemblyjs/utf8": "1.8.5" + } + }, + "node_modules/@webassemblyjs/wast-parser": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.8.5.tgz", + "integrity": "sha512-daXC1FyKWHF1i11obK086QRlsMsY4+tIOKgBqI1lxAnkp9xe9YMcgOxm9kLe+ttjs5aWV2KKE1TWJCN57/Btsg==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/floating-point-hex-parser": "1.8.5", + "@webassemblyjs/helper-api-error": "1.8.5", + "@webassemblyjs/helper-code-frame": "1.8.5", + "@webassemblyjs/helper-fsm": "1.8.5", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.8.5.tgz", + "integrity": "sha512-w0U0pD4EhlnvRyeJzBqaVSJAo9w/ce7/WPogeXLzGkO6hzhr4GnQIZ4W4uUt5b9ooAaXPtnXlj0gzsXEOUNYMg==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/wast-parser": "1.8.5", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" + }, + "node_modules/abab": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.4.tgz", + "integrity": "sha512-Eu9ELJWCz/c1e9gTiCY+FceWxcqzjYEbqMgtndnuSqZSUCOL73TWNK2mHfIj4Cw2E/ongOp+JISVNCmovt2KYQ==" + }, + "node_modules/accepts": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", + "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", + "dependencies": { + "mime-types": "~2.1.24", + "negotiator": "0.6.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-globals": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.4.tgz", + "integrity": "sha512-clfQEh21R+D0leSbUdWf3OcfqyaCSAQ8Ryq00bofSekfr9W8u1jyYZo6ir0xu9Gtcf7BjcHJpnbZH7JOCpP60A==", + "dependencies": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, + "node_modules/acorn-globals/node_modules/acorn": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==" + }, + "node_modules/acorn-walk": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", + "integrity": "sha512-7evsyfH1cLOCdAzZAd43Cic04yKydNx0cF+7tiA19p1XnLLPU4dpCQOqpjqwokFe//vS0QqfqqjCS2JkiIs0cA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/address": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/address/-/address-1.1.2.tgz", + "integrity": "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==", + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/adjust-sourcemap-loader": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-3.0.0.tgz", + "integrity": "sha512-YBrGyT2/uVQ/c6Rr+t6ZJXniY03YtHGMJQYal368burRGYKqhx9qGTWqcBU5s1CwYY9E/ri63RYyG1IacMZtqw==", + "dependencies": { + "loader-utils": "^2.0.0", + "regex-parser": "^2.2.11" + }, + "engines": { + "node": ">=8.9" + } + }, + "node_modules/adjust-sourcemap-loader/node_modules/loader-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ajv": { + "version": "6.12.4", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.4.tgz", + "integrity": "sha512-eienB2c9qVQs2KWexhkrdMLVDoIQCz5KSeLxwg9Lzk4DOfBtIK9PQwwufcsn1jjGuf9WZmqPMbGxOzfcuphJCQ==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/ajv-errors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", + "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==" + }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==" + }, + "node_modules/alphanum-sort": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz", + "integrity": "sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM=" + }, + "node_modules/ansi-colors": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.4.tgz", + "integrity": "sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.1.tgz", + "integrity": "sha512-JWF7ocqNrp8u9oqpgV+wH5ftbt+cfvv+PTjOvKLT3AdYly/LmORARfEVT1iyjwN+4MqE5UmVKoAdIBqeoCHgLA==", + "dependencies": { + "type-fest": "^0.11.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.11.0.tgz", + "integrity": "sha512-OdjXJxnCN1AvyLSzeKIgXTXxV+99ZuXl3Hpo9XpJAv9MBcHrrJOQ5kV7ypXOuQie+AmWG25hLbiKdwYTifzcfQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-html": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz", + "integrity": "sha1-gTWEAhliqenm/QOflA0S9WynhZ4=", + "engines": [ + "node >= 0.8.0" + ], + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", + "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "dependencies": { + "micromatch": "^3.1.4", + "normalize-path": "^2.1.1" + } + }, + "node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/aria-query": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz", + "integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=", + "dependencies": { + "ast-types-flow": "0.0.7", + "commander": "^2.11.0" + } + }, + "node_modules/arity-n": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/arity-n/-/arity-n-1.0.4.tgz", + "integrity": "sha1-2edrEXM+CFacCEeuezmyhgswt0U=" + }, + "node_modules/arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, + "node_modules/array-flatten": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", + "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==" + }, + "node_modules/array-includes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", + "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "is-string": "^1.0.5" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array.prototype.flat": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", + "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + }, + "node_modules/asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/asn1.js/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/assert": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/assert/-/assert-1.4.1.tgz", + "integrity": "sha1-mZEtWRg2tab1s0XA8H7vwI/GXZE=", + "dependencies": { + "util": "0.10.3" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ast-types-flow": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", + "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=" + }, + "node_modules/astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "engines": { + "node": ">=4" + } + }, + "node_modules/async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/async-each": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", + "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==" + }, + "node_modules/async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "bin": { + "atob": "bin/atob.js" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/autoprefixer": { + "version": "9.8.6", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-9.8.6.tgz", + "integrity": "sha512-XrvP4VVHdRBCdX1S3WXVD8+RyG9qeb1D5Sn1DeLiG2xfSpzellk5k54xbUERJ3M5DggQxes39UGOTP8CFrEGbg==", + "dependencies": { + "browserslist": "^4.12.0", + "caniuse-lite": "^1.0.30001109", + "colorette": "^1.2.1", + "normalize-range": "^0.1.2", + "num2fraction": "^1.2.2", + "postcss": "^7.0.32", + "postcss-value-parser": "^4.1.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + } + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "engines": { + "node": "*" + } + }, + "node_modules/aws4": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.1.tgz", + "integrity": "sha512-zg7Hz2k5lI8kb7U32998pRRFin7zJlkfezGJjUc2heaD4Pw2wObakCDVzkKztTm/Ln7eiVvYsjqak0Ed4LkMDA==" + }, + "node_modules/axobject-query": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz", + "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==" + }, + "node_modules/babel-code-frame": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", + "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", + "dependencies": { + "chalk": "^1.1.3", + "esutils": "^2.0.2", + "js-tokens": "^3.0.2" + } + }, + "node_modules/babel-code-frame/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/js-tokens": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", + "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=" + }, + "node_modules/babel-code-frame/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/babel-code-frame/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/babel-eslint": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", + "integrity": "sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.7.0", + "@babel/traverse": "^7.7.0", + "@babel/types": "^7.7.0", + "eslint-visitor-keys": "^1.0.0", + "resolve": "^1.12.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/babel-extract-comments": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/babel-extract-comments/-/babel-extract-comments-1.0.0.tgz", + "integrity": "sha512-qWWzi4TlddohA91bFwgt6zO/J0X+io7Qp184Fw0m2JYRSTZnJbFR8+07KmzudHCZgOiKRCrjhylwv9Xd8gfhVQ==", + "dependencies": { + "babylon": "^6.18.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-jest": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-24.9.0.tgz", + "integrity": "sha512-ntuddfyiN+EhMw58PTNL1ph4C9rECiQXjI4nMMBKBaNjXvqLdkXpPRcMSr4iyBrJg/+wz9brFUD6RhOAT6r4Iw==", + "dependencies": { + "@jest/transform": "^24.9.0", + "@jest/types": "^24.9.0", + "@types/babel__core": "^7.1.0", + "babel-plugin-istanbul": "^5.1.0", + "babel-preset-jest": "^24.9.0", + "chalk": "^2.4.2", + "slash": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/babel-jest/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-jest/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-jest/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/babel-jest/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/babel-jest/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-jest/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-loader": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", + "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", + "dependencies": { + "find-cache-dir": "^2.1.0", + "loader-utils": "^1.4.0", + "mkdirp": "^0.5.3", + "pify": "^4.0.1", + "schema-utils": "^2.6.5" + }, + "engines": { + "node": ">= 6.9" + } + }, + "node_modules/babel-loader/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dependencies": { + "object.assign": "^4.1.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-5.2.0.tgz", + "integrity": "sha512-5LphC0USA8t4i1zCtjbbNb6jJj/9+X6P37Qfirc/70EQ34xKlMW+a1RHGwxGI+SwWpNwZ27HqvzAobeqaXwiZw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "find-up": "^3.0.0", + "istanbul-lib-instrument": "^3.3.0", + "test-exclude": "^5.2.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-24.9.0.tgz", + "integrity": "sha512-2EMA2P8Vp7lG0RAzr4HXqtYwacfMErOuv1U3wrvxHX6rD1sV6xS3WXG3r8TRQ2r6w8OhvSdWt+z41hQNwNm3Xw==", + "dependencies": { + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/babel-plugin-macros": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-2.8.0.tgz", + "integrity": "sha512-SEP5kJpfGYqYKpBrj5XU3ahw5p5GOHJ0U5ssOSQ/WBVdwkD2Dzlce95exQTs3jOVWPPKLBN2rlEWkCK7dSmLvg==", + "dependencies": { + "@babel/runtime": "^7.7.2", + "cosmiconfig": "^6.0.0", + "resolve": "^1.12.0" + } + }, + "node_modules/babel-plugin-macros/node_modules/cosmiconfig": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz", + "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.1.0", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.7.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-macros/node_modules/import-fresh": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/babel-plugin-macros/node_modules/parse-json": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz", + "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-macros/node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-macros/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "engines": { + "node": ">=4" + } + }, + "node_modules/babel-plugin-named-asset-import": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/babel-plugin-named-asset-import/-/babel-plugin-named-asset-import-0.3.6.tgz", + "integrity": "sha512-1aGDUfL1qOOIoqk9QKGIo2lANk+C7ko/fqH0uIyC71x3PEGz0uVP8ISgfEsFuG+FKmjHTvFK/nNM8dowpmUxLA==" + }, + "node_modules/babel-plugin-syntax-object-rest-spread": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz", + "integrity": "sha1-/WU28rzhODb/o6VFjEkDpZe7O/U=" + }, + "node_modules/babel-plugin-transform-object-rest-spread": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-object-rest-spread/-/babel-plugin-transform-object-rest-spread-6.26.0.tgz", + "integrity": "sha1-DzZpLVD+9rfi1LOsFHgTepY7ewY=", + "dependencies": { + "babel-plugin-syntax-object-rest-spread": "^6.8.0", + "babel-runtime": "^6.26.0" + } + }, + "node_modules/babel-plugin-transform-react-remove-prop-types": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-react-remove-prop-types/-/babel-plugin-transform-react-remove-prop-types-0.4.24.tgz", + "integrity": "sha512-eqj0hVcJUR57/Ug2zE1Yswsw4LhuqqHhD+8v120T1cl3kjg76QwtyBrdIk4WVwK+lAhBJVYCd/v+4nc4y+8JsA==" + }, + "node_modules/babel-preset-jest": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-24.9.0.tgz", + "integrity": "sha512-izTUuhE4TMfTRPF92fFwD2QfdXaZW08qvWTFCI51V8rW5x00UuPgc3ajRoWofXOuxjfcOM5zzSYsQS3H8KGCAg==", + "dependencies": { + "@babel/plugin-syntax-object-rest-spread": "^7.0.0", + "babel-plugin-jest-hoist": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/babel-preset-react-app": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/babel-preset-react-app/-/babel-preset-react-app-9.1.2.tgz", + "integrity": "sha512-k58RtQOKH21NyKtzptoAvtAODuAJJs3ZhqBMl456/GnXEQ/0La92pNmwgWoMn5pBTrsvk3YYXdY7zpY4e3UIxA==", + "dependencies": { + "@babel/core": "7.9.0", + "@babel/plugin-proposal-class-properties": "7.8.3", + "@babel/plugin-proposal-decorators": "7.8.3", + "@babel/plugin-proposal-nullish-coalescing-operator": "7.8.3", + "@babel/plugin-proposal-numeric-separator": "7.8.3", + "@babel/plugin-proposal-optional-chaining": "7.9.0", + "@babel/plugin-transform-flow-strip-types": "7.9.0", + "@babel/plugin-transform-react-display-name": "7.8.3", + "@babel/plugin-transform-runtime": "7.9.0", + "@babel/preset-env": "7.9.0", + "@babel/preset-react": "7.9.1", + "@babel/preset-typescript": "7.9.0", + "@babel/runtime": "7.9.0", + "babel-plugin-macros": "2.8.0", + "babel-plugin-transform-react-remove-prop-types": "0.4.24" + } + }, + "node_modules/babel-preset-react-app/node_modules/@babel/plugin-proposal-class-properties": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.8.3.tgz", + "integrity": "sha512-EqFhbo7IosdgPgZggHaNObkmO1kNUe3slaKu54d5OWvy+p9QIKOzK1GAEpAIsZtWVtPXUHSMcT4smvDrCfY4AA==", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3" + } + }, + "node_modules/babel-preset-react-app/node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-TS9MlfzXpXKt6YYomudb/KU7nQI6/xnapG6in1uZxoxDghuSMZsPb6D2fyUwNYSAp4l1iR7QtFOjkqcRYcUsfw==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0" + } + }, + "node_modules/babel-preset-react-app/node_modules/@babel/plugin-proposal-numeric-separator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.8.3.tgz", + "integrity": "sha512-jWioO1s6R/R+wEHizfaScNsAx+xKgwTLNXSh7tTC4Usj3ItsPEhYkEpU4h+lpnBwq7NBVOJXfO6cRFYcX69JUQ==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.8.3" + } + }, + "node_modules/babel-preset-react-app/node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.9.0.tgz", + "integrity": "sha512-NDn5tu3tcv4W30jNhmc2hyD5c56G6cXx4TesJubhxrJeCvuuMpttxr0OnNCqbZGhFjLrg+NIhxxC+BK5F6yS3w==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.0" + } + }, + "node_modules/babel-preset-react-app/node_modules/@babel/plugin-transform-react-display-name": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.8.3.tgz", + "integrity": "sha512-3Jy/PCw8Fe6uBKtEgz3M82ljt+lTg+xJaM4og+eyu83qLT87ZUSckn0wy7r31jflURWLO83TW6Ylf7lyXj3m5A==", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3" + } + }, + "node_modules/babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", + "dependencies": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, + "node_modules/babel-runtime/node_modules/core-js": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==", + "deprecated": "core-js@<3 is no longer maintained and not recommended for usage due to the number of issues. Please, upgrade your dependencies to the actual version of core-js@3.", + "hasInstallScript": true + }, + "node_modules/babel-runtime/node_modules/regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + }, + "node_modules/babylon": { + "version": "6.18.0", + "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", + "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==", + "bin": { + "babylon": "bin/babylon.js" + } + }, + "node_modules/balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "node_modules/base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dependencies": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", + "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "node_modules/bn.js": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz", + "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==" + }, + "node_modules/body-parser": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", + "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", + "dependencies": { + "bytes": "3.1.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.7.0", + "raw-body": "2.4.0", + "type-is": "~1.6.17" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/bonjour": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz", + "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=", + "dependencies": { + "array-flatten": "^2.1.0", + "deep-equal": "^1.0.1", + "dns-equal": "^1.0.0", + "dns-txt": "^2.0.2", + "multicast-dns": "^6.0.1", + "multicast-dns-service-types": "^1.1.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + }, + "node_modules/browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" + }, + "node_modules/browser-resolve": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.3.tgz", + "integrity": "sha512-exDi1BYWB/6raKHmDTCicQfTkqwN5fioMFV4j8BsfMU4R2DK/QfZfK7kOVkmWCNANf0snkBzqGqAJBao9gZMdQ==", + "dependencies": { + "resolve": "1.1.7" + } + }, + "node_modules/browser-resolve/node_modules/resolve": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=" + }, + "node_modules/browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "dependencies": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "dependencies": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "node_modules/browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "dependencies": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/browserify-rsa": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", + "dependencies": { + "bn.js": "^4.1.0", + "randombytes": "^2.0.1" + } + }, + "node_modules/browserify-rsa/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "dependencies": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "node_modules/browserify-sign/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "node_modules/browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "dependencies": { + "pako": "~1.0.5" + } + }, + "node_modules/browserslist": { + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.15.0.tgz", + "integrity": "sha512-IJ1iysdMkGmjjYeRlDU8PQejVwxvVO5QOfXH7ylW31GO6LwNRSmm/SgRXtNsEXqMLl2e+2H5eEJ7sfynF8TCaQ==", + "dependencies": { + "caniuse-lite": "^1.0.30001164", + "colorette": "^1.2.1", + "electron-to-chromium": "^1.3.612", + "escalade": "^3.1.1", + "node-releases": "^1.1.67" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "node_modules/buffer-indexof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-indexof/-/buffer-indexof-1.1.1.tgz", + "integrity": "sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g==" + }, + "node_modules/buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" + }, + "node_modules/buffer/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/builtin-status-codes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", + "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" + }, + "node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacache": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-13.0.1.tgz", + "integrity": "sha512-5ZvAxd05HDDU+y9BVvcqYu2LLXmPnQ0hW62h32g4xBTgL/MppR4/04NHfj/ycM2y6lmTnbw6HVi+1eN0Psba6w==", + "dependencies": { + "chownr": "^1.1.2", + "figgy-pudding": "^3.5.1", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.2", + "infer-owner": "^1.0.4", + "lru-cache": "^5.1.1", + "minipass": "^3.0.0", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "p-map": "^3.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^2.7.1", + "ssri": "^7.0.0", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cacache/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dependencies": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/call-bind": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.0.tgz", + "integrity": "sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-me-maybe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz", + "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=" + }, + "node_modules/caller-callsite": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/caller-callsite/-/caller-callsite-2.0.0.tgz", + "integrity": "sha1-hH4PzgoiN1CpoCfFSzNzGtMVQTQ=", + "dependencies": { + "callsites": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/caller-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-2.0.0.tgz", + "integrity": "sha1-Ro+DBE42mrIBD6xfBs7uFbsssfQ=", + "dependencies": { + "caller-callsite": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/callsites": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "integrity": "sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=", + "engines": { + "node": ">=4" + } + }, + "node_modules/camel-case": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.1.tgz", + "integrity": "sha512-7fa2WcG4fYFkclIvEmxBbTvmibwF2/agfEBc6q3lOpVu0A13ltLsA+Hr/8Hp6kp5f+G7hKi6t8lys6XxP+1K6Q==", + "dependencies": { + "pascal-case": "^3.1.1", + "tslib": "^1.10.0" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz", + "integrity": "sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==", + "dependencies": { + "browserslist": "^4.0.0", + "caniuse-lite": "^1.0.0", + "lodash.memoize": "^4.1.2", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001165", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001165.tgz", + "integrity": "sha512-8cEsSMwXfx7lWSUMA2s08z9dIgsnR5NAqjXP23stdsU3AUWkCr/rr4s4OFtHXn5XXr6+7kam3QFVoYyXNPdJPA==" + }, + "node_modules/capture-exit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/capture-exit/-/capture-exit-2.0.0.tgz", + "integrity": "sha512-PiT/hQmTonHhl/HFGN+Lx3JJUznrVYJ3+AQsnthneZbvW7x+f08Tk7yLJTLEOUvBTbduLeeBkxEaYXUOUrRq6g==", + "dependencies": { + "rsvp": "^4.8.4" + }, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/case-sensitive-paths-webpack-plugin": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/case-sensitive-paths-webpack-plugin/-/case-sensitive-paths-webpack-plugin-2.3.0.tgz", + "integrity": "sha512-/4YgnZS8y1UXXmC02xD5rRrBEu6T5ub+mQHLNRj0fzTRbgdBYhsNo2V5EqwgqrExjxsjtF/OpAKAMkKsxbD5XQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" + }, + "node_modules/chokidar": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz", + "integrity": "sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==", + "dependencies": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.5.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.1.2" + } + }, + "node_modules/chokidar/node_modules/anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/chokidar/node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar/node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar/node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/chokidar/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/chrome-trace-event": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz", + "integrity": "sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ==", + "dependencies": { + "tslib": "^1.9.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==" + }, + "node_modules/cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dependencies": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/classnames": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.2.6.tgz", + "integrity": "sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q==" + }, + "node_modules/clean-css": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz", + "integrity": "sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 4.0" + } + }, + "node_modules/clean-css/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" + }, + "node_modules/cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dependencies": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "engines": { + "node": ">=4" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha1-TnPdCen7lxzDhnDF3O2cGJZIHMY=", + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clsx": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.1.1.tgz", + "integrity": "sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/coa": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/coa/-/coa-2.0.2.tgz", + "integrity": "sha512-q5/jG+YQnSy4nRTV4F7lPepBJZ8qBNJJDBuJdoejDyLXgmL7IEo+Le2JDZudFTFt7mrCqIRaSjws4ygRCTCAXA==", + "dependencies": { + "@types/q": "^1.5.1", + "chalk": "^2.4.1", + "q": "^1.1.2" + }, + "engines": { + "node": ">= 4.0" + } + }, + "node_modules/coa/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/coa/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/coa/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/coa/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/coa/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/coa/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=", + "dependencies": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/color/-/color-3.1.2.tgz", + "integrity": "sha512-vXTJhHebByxZn3lDvDJYw4lR5+uB3vuoHsuYA5AKuxRVn5wzzIfQKGLBmgdVRHKTJYeK5rvJcHnrd0Li49CFpg==", + "dependencies": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz", + "integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/colorette": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", + "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "node_modules/common-tags": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", + "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=" + }, + "node_modules/component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" + }, + "node_modules/compose-function": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/compose-function/-/compose-function-3.0.3.tgz", + "integrity": "sha1-ntZ18TzFRQHTCVCkhv9qe6OrGF8=", + "dependencies": { + "arity-n": "^1.0.4" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", + "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", + "dependencies": { + "accepts": "~1.3.5", + "bytes": "3.0.0", + "compressible": "~2.0.16", + "debug": "2.6.9", + "on-headers": "~1.0.2", + "safe-buffer": "5.1.2", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/concat-stream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/confusing-browser-globals": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/confusing-browser-globals/-/confusing-browser-globals-1.0.9.tgz", + "integrity": "sha512-KbS1Y0jMtyPgIxjO7ZzMAuUpAKMt1SzCL9fsrKsX6b0zJPTaT0SiSPmewwVZg9UAO83HVIlEhZF84LIjZ0lmAw==" + }, + "node_modules/connect-history-api-fallback": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-1.6.0.tgz", + "integrity": "sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/console-browserify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + }, + "node_modules/constants-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", + "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=" + }, + "node_modules/contains-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", + "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/content-disposition": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", + "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", + "integrity": "sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==", + "dependencies": { + "safe-buffer": "~5.1.1" + } + }, + "node_modules/cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "node_modules/copy-concurrently": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", + "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "dependencies": { + "aproba": "^1.1.1", + "fs-write-stream-atomic": "^1.0.8", + "iferr": "^0.1.5", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.0" + } + }, + "node_modules/copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/core-js": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.8.0.tgz", + "integrity": "sha512-W2VYNB0nwQQE7tKS7HzXd7r2y/y2SVJl4ga6oH/dnaLFzM0o2lB2P3zCkWj5Wc/zyMYjtgd5Hmhk0ObkQFZOIA==", + "hasInstallScript": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-compat": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.8.0.tgz", + "integrity": "sha512-o9QKelQSxQMYWHXc/Gc4L8bx/4F7TTraE5rhuN8I7mKBt5dBIUpXpIR3omv70ebr8ST5R3PqbDQr+ZI3+Tt1FQ==", + "dependencies": { + "browserslist": "^4.14.7", + "semver": "7.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-compat/node_modules/semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/core-js-pure": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.6.5.tgz", + "integrity": "sha512-lacdXOimsiD0QyNf9BC/mxivNJ/ybBGJXQFKzRekp1WTHoVUWsUHEn+2T8GJAzzIhyOuXA+gOxCVN3l+5PLPUA==", + "hasInstallScript": true + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "node_modules/cosmiconfig": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.2.1.tgz", + "integrity": "sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==", + "dependencies": { + "import-fresh": "^2.0.0", + "is-directory": "^0.3.1", + "js-yaml": "^3.13.1", + "parse-json": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "dependencies": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + } + }, + "node_modules/create-ecdh/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "node_modules/create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "dependencies": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "node_modules/cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/cross-spawn/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "dependencies": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + }, + "engines": { + "node": "*" + } + }, + "node_modules/css": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/css/-/css-2.2.4.tgz", + "integrity": "sha512-oUnjmWpy0niI3x/mPL8dVEI1l7MnG3+HHyRPHf+YFSbK+svOhXpmSOcDURUh2aOCgl2grzrOPt1nHLuCVFULLw==", + "dependencies": { + "inherits": "^2.0.3", + "source-map": "^0.6.1", + "source-map-resolve": "^0.5.2", + "urix": "^0.1.0" + } + }, + "node_modules/css-blank-pseudo": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/css-blank-pseudo/-/css-blank-pseudo-0.1.4.tgz", + "integrity": "sha512-LHz35Hr83dnFeipc7oqFDmsjHdljj3TQtxGGiNWSOsTLIAubSm4TEz8qCaKFpk7idaQ1GfWscF4E6mgpBysA1w==", + "dependencies": { + "postcss": "^7.0.5" + }, + "bin": { + "css-blank-pseudo": "cli.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/css-color-names": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz", + "integrity": "sha1-gIrcLnnPhHOAabZGyyDsJ762KeA=", + "engines": { + "node": "*" + } + }, + "node_modules/css-declaration-sorter": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-4.0.1.tgz", + "integrity": "sha512-BcxQSKTSEEQUftYpBVnsH4SF05NTuBokb19/sBt6asXGKZ/6VP7PLG1CBCkFDYOnhXhPh0jMhO6xZ71oYHXHBA==", + "dependencies": { + "postcss": "^7.0.1", + "timsort": "^0.3.0" + }, + "engines": { + "node": ">4" + } + }, + "node_modules/css-has-pseudo": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/css-has-pseudo/-/css-has-pseudo-0.10.0.tgz", + "integrity": "sha512-Z8hnfsZu4o/kt+AuFzeGpLVhFOGO9mluyHBaA2bA8aCGTwah5sT3WV/fTHH8UNZUytOIImuGPrl/prlb4oX4qQ==", + "dependencies": { + "postcss": "^7.0.6", + "postcss-selector-parser": "^5.0.0-rc.4" + }, + "bin": { + "css-has-pseudo": "cli.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/css-has-pseudo/node_modules/cssesc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-2.0.0.tgz", + "integrity": "sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/css-has-pseudo/node_modules/postcss-selector-parser": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz", + "integrity": "sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ==", + "dependencies": { + "cssesc": "^2.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/css-loader": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-3.4.2.tgz", + "integrity": "sha512-jYq4zdZT0oS0Iykt+fqnzVLRIeiPWhka+7BqPn+oSIpWJAHak5tmB/WZrJ2a21JhCeFyNnnlroSl8c+MtVndzA==", + "dependencies": { + "camelcase": "^5.3.1", + "cssesc": "^3.0.0", + "icss-utils": "^4.1.1", + "loader-utils": "^1.2.3", + "normalize-path": "^3.0.0", + "postcss": "^7.0.23", + "postcss-modules-extract-imports": "^2.0.0", + "postcss-modules-local-by-default": "^3.0.2", + "postcss-modules-scope": "^2.1.1", + "postcss-modules-values": "^3.0.0", + "postcss-value-parser": "^4.0.2", + "schema-utils": "^2.6.0" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/css-loader/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/css-prefers-color-scheme": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/css-prefers-color-scheme/-/css-prefers-color-scheme-3.1.1.tgz", + "integrity": "sha512-MTu6+tMs9S3EUqzmqLXEcgNRbNkkD/TGFvowpeoWJn5Vfq7FMgsmRQs9X5NXAURiOBmOxm/lLjsDNXDE6k9bhg==", + "dependencies": { + "postcss": "^7.0.5" + }, + "bin": { + "css-prefers-color-scheme": "cli.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/css-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-2.1.0.tgz", + "integrity": "sha512-Dqk7LQKpwLoH3VovzZnkzegqNSuAziQyNZUcrdDM401iY+R5NkGBXGmtO05/yaXQziALuPogeG0b7UAgjnTJTQ==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^3.2.1", + "domutils": "^1.7.0", + "nth-check": "^1.0.2" + } + }, + "node_modules/css-select-base-adapter": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz", + "integrity": "sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w==" + }, + "node_modules/css-tree": { + "version": "1.0.0-alpha.37", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.37.tgz", + "integrity": "sha512-DMxWJg0rnz7UgxKT0Q1HU/L9BeJI0M6ksor0OgqOnF+aRCDWg/N2641HmVyU9KVIu0OVVWOb2IpC9A+BJRnejg==", + "dependencies": { + "mdn-data": "2.0.4", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/css-tree/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/css-vendor": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/css-vendor/-/css-vendor-2.0.8.tgz", + "integrity": "sha512-x9Aq0XTInxrkuFeHKbYC7zWY8ai7qJ04Kxd9MnvbC1uO5DagxoHQjm4JvG+vCdXOoFtCjbL2XSZfxmoYa9uQVQ==", + "dependencies": { + "@babel/runtime": "^7.8.3", + "is-in-browser": "^1.0.2" + } + }, + "node_modules/css-what": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", + "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cssdb": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/cssdb/-/cssdb-4.4.0.tgz", + "integrity": "sha512-LsTAR1JPEM9TpGhl/0p3nQecC2LJ0kD8X5YARu1hk/9I1gril5vDtMZyNxcEpxxDj34YNck/ucjuoUd66K03oQ==" + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cssnano": { + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-4.1.10.tgz", + "integrity": "sha512-5wny+F6H4/8RgNlaqab4ktc3e0/blKutmq8yNlBFXA//nSFFAqAngjNVRzUvCgYROULmZZUoosL/KSoZo5aUaQ==", + "dependencies": { + "cosmiconfig": "^5.0.0", + "cssnano-preset-default": "^4.0.7", + "is-resolvable": "^1.0.0", + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/cssnano-preset-default": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/cssnano-preset-default/-/cssnano-preset-default-4.0.7.tgz", + "integrity": "sha512-x0YHHx2h6p0fCl1zY9L9roD7rnlltugGu7zXSKQx6k2rYw0Hi3IqxcoAGF7u9Q5w1nt7vK0ulxV8Lo+EvllGsA==", + "dependencies": { + "css-declaration-sorter": "^4.0.1", + "cssnano-util-raw-cache": "^4.0.1", + "postcss": "^7.0.0", + "postcss-calc": "^7.0.1", + "postcss-colormin": "^4.0.3", + "postcss-convert-values": "^4.0.1", + "postcss-discard-comments": "^4.0.2", + "postcss-discard-duplicates": "^4.0.2", + "postcss-discard-empty": "^4.0.1", + "postcss-discard-overridden": "^4.0.1", + "postcss-merge-longhand": "^4.0.11", + "postcss-merge-rules": "^4.0.3", + "postcss-minify-font-values": "^4.0.2", + "postcss-minify-gradients": "^4.0.2", + "postcss-minify-params": "^4.0.2", + "postcss-minify-selectors": "^4.0.2", + "postcss-normalize-charset": "^4.0.1", + "postcss-normalize-display-values": "^4.0.2", + "postcss-normalize-positions": "^4.0.2", + "postcss-normalize-repeat-style": "^4.0.2", + "postcss-normalize-string": "^4.0.2", + "postcss-normalize-timing-functions": "^4.0.2", + "postcss-normalize-unicode": "^4.0.1", + "postcss-normalize-url": "^4.0.1", + "postcss-normalize-whitespace": "^4.0.2", + "postcss-ordered-values": "^4.1.2", + "postcss-reduce-initial": "^4.0.3", + "postcss-reduce-transforms": "^4.0.2", + "postcss-svgo": "^4.0.2", + "postcss-unique-selectors": "^4.0.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/cssnano-util-get-arguments": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cssnano-util-get-arguments/-/cssnano-util-get-arguments-4.0.0.tgz", + "integrity": "sha1-7ToIKZ8h11dBsg87gfGU7UnMFQ8=", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/cssnano-util-get-match": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cssnano-util-get-match/-/cssnano-util-get-match-4.0.0.tgz", + "integrity": "sha1-wOTKB/U4a7F+xeUiULT1lhNlFW0=", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/cssnano-util-raw-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/cssnano-util-raw-cache/-/cssnano-util-raw-cache-4.0.1.tgz", + "integrity": "sha512-qLuYtWK2b2Dy55I8ZX3ky1Z16WYsx544Q0UWViebptpwn/xDBmog2TLg4f+DBMg1rJ6JDWtn96WHbOKDWt1WQA==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/cssnano-util-same-parent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/cssnano-util-same-parent/-/cssnano-util-same-parent-4.0.1.tgz", + "integrity": "sha512-WcKx5OY+KoSIAxBW6UBBRay1U6vkYheCdjyVNDm85zt5K9mHoGOfsOsqIszfAqrQQFIIKgjh2+FDgIj/zsl21Q==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/csso": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/csso/-/csso-4.0.3.tgz", + "integrity": "sha512-NL3spysxUkcrOgnpsT4Xdl2aiEiBG6bXswAABQVHcMrfjjBisFOKwLDOmf4wf32aPdcJws1zds2B0Rg+jqMyHQ==", + "dependencies": { + "css-tree": "1.0.0-alpha.39" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/csso/node_modules/css-tree": { + "version": "1.0.0-alpha.39", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.39.tgz", + "integrity": "sha512-7UvkEYgBAHRG9Nt980lYxjsTrCyHFN53ky3wVsDkiMdVqylqRt+Zc+jm5qw7/qyOvN2dHSYtX0e4MbCCExSvnA==", + "dependencies": { + "mdn-data": "2.0.6", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/csso/node_modules/mdn-data": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.6.tgz", + "integrity": "sha512-rQvjv71olwNHgiTbfPZFkJtjNMciWgswYeciZhtvWLO8bmX3TnhyA62I6sTWOyZssWHJJjY6/KiWwqQsWWsqOA==" + }, + "node_modules/csso/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + }, + "node_modules/cssstyle": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.4.0.tgz", + "integrity": "sha512-GBrLZYZ4X4x6/QEoBnIrqb8B/f5l4+8me2dkom/j1Gtbxy0kBv6OGzKuAsGM75bkGwGAFkt56Iwg28S3XTZgSA==", + "dependencies": { + "cssom": "0.3.x" + } + }, + "node_modules/csstype": { + "version": "2.6.14", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.14.tgz", + "integrity": "sha512-2mSc+VEpGPblzAxyeR+vZhJKgYg0Og0nnRi7pmRXFYYxSfnOnW8A5wwQb4n4cE2nIOzqKOAzLCaEX6aBmNEv8A==" + }, + "node_modules/cyclist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", + "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=" + }, + "node_modules/d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dependencies": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + }, + "node_modules/damerau-levenshtein": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.6.tgz", + "integrity": "sha512-JVrozIeElnj3QzfUIt8tB8YMluBJom4Vw9qTPpjGYQ9fYlB3D/rb6OordUxf3xeFB35LKWs0xqcO5U6ySvBtug==" + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dependencies": { + "assert-plus": "^1.0.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "dependencies": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, + "node_modules/data-urls/node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/deep-equal": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz", + "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", + "dependencies": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } + }, + "node_modules/deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "node_modules/default-gateway": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-4.2.0.tgz", + "integrity": "sha512-h6sMrVB1VMWVrW13mSc6ia/DwYYw5MN6+exNu1OaJeFac5aSAvwM7lZ0NVfTABuSkQelr4h5oebg3KB1XPdjgA==", + "dependencies": { + "execa": "^1.0.0", + "ip-regex": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dependencies": { + "object-keys": "^1.0.12" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dependencies": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", + "dependencies": { + "@types/glob": "^7.1.1", + "globby": "^6.1.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/del/node_modules/globby": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", + "dependencies": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del/node_modules/globby/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del/node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/del/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/des.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", + "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "dependencies": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/destroy": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" + }, + "node_modules/detect-newline": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-2.1.0.tgz", + "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-node": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.0.4.tgz", + "integrity": "sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==" + }, + "node_modules/detect-port-alt": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/detect-port-alt/-/detect-port-alt-1.1.6.tgz", + "integrity": "sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q==", + "dependencies": { + "address": "^1.0.1", + "debug": "^2.6.0" + }, + "bin": { + "detect": "bin/detect-port", + "detect-port": "bin/detect-port" + }, + "engines": { + "node": ">= 4.2.1" + } + }, + "node_modules/detect-port-alt/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/detect-port-alt/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/diff-sequences": { + "version": "25.2.6", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz", + "integrity": "sha512-Hq8o7+6GaZeoFjtpgvRBUknSXNeJiCx7V9Fr94ZMljNiCr9n9L8H8aJqgWOQiDDGdyn29fRNcDdRVJ5fdyihfg==", + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "dependencies": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + } + }, + "node_modules/diffie-hellman/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/dir-glob": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz", + "integrity": "sha512-37qirFDz8cA5fimp9feo43fSuRo2gHwaIn6dXL8Ber1dGwUosDrGZeCCXq57WnIqE4aQ+u3eQZzsk1yOzhdwag==", + "dependencies": { + "arrify": "^1.0.1", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/dns-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", + "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=" + }, + "node_modules/dns-packet": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-1.3.1.tgz", + "integrity": "sha512-0UxfQkMhYAUaZI+xrNZOz/as5KgDU0M/fQ9b6SpkyLbk3GEswDi6PADJVaYJradtRVsRIlF1zLyOodbcTCDzUg==", + "dependencies": { + "ip": "^1.1.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/dns-txt": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz", + "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=", + "dependencies": { + "buffer-indexof": "^1.0.0" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "dependencies": { + "utila": "~0.4" + } + }, + "node_modules/dom-helpers": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.0.tgz", + "integrity": "sha512-Ru5o9+V8CpunKnz5LGgWXkmrH/20cGKwcHwS4m73zIvs54CN9epEmT/HLqFJW3kXpakAFkEdzgy1hzlJe3E4OQ==", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "node_modules/dom-helpers/node_modules/csstype": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.5.tgz", + "integrity": "sha512-uVDi8LpBUKQj6sdxNaTetL6FpeCqTjOvAQuQUa/qAqq8oOd4ivkbhgnqayl0dnPal8Tb/yB1tF+gOvCBiicaiQ==" + }, + "node_modules/dom-serializer": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.2.tgz", + "integrity": "sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==", + "dependencies": { + "domelementtype": "^2.0.1", + "entities": "^2.0.0" + } + }, + "node_modules/dom-serializer/node_modules/domelementtype": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.1.0.tgz", + "integrity": "sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "engines": { + "node": ">=0.4", + "npm": ">=1.2" + } + }, + "node_modules/domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "node_modules/domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "dependencies": { + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dependencies": { + "domelementtype": "1" + } + }, + "node_modules/domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/dot-case": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.3.tgz", + "integrity": "sha512-7hwEmg6RiSQfm/GwPL4AAWXKy3YNNZA3oFv2Pdiey0mwkRCPZ9x6SZbkLcn8Ma5PYeVokzoD4Twv2n7LKp5WeA==", + "dependencies": { + "no-case": "^3.0.3", + "tslib": "^1.10.0" + } + }, + "node_modules/dot-prop": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", + "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv-expand": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", + "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==" + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, + "node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/duplexify/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/duplexify/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + }, + "node_modules/electron-to-chromium": { + "version": "1.3.615", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.615.tgz", + "integrity": "sha512-fNYTQXoUhNc6RmHDlGN4dgcLURSBIqQCN7ls6MuQ741+NJyLNRz8DxAC+pZpOKfRs6cfY0lv2kWdy8Oxf9j4+A==" + }, + "node_modules/elliptic": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "dependencies": { + "bn.js": "^4.4.0", + "brorand": "^1.0.1", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.0" + } + }, + "node_modules/elliptic/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enhanced-resolve": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz", + "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==", + "dependencies": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.5.0", + "tapable": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/enhanced-resolve/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/enhanced-resolve/node_modules/memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dependencies": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + }, + "engines": { + "node": ">=4.3.0 <5.0.0 || >=5.10" + } + }, + "node_modules/enhanced-resolve/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/entities": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/errno": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es5-ext": { + "version": "0.10.53", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", + "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", + "dependencies": { + "es6-iterator": "~2.0.3", + "es6-symbol": "~3.1.3", + "next-tick": "~1.0.0" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "dependencies": { + "d": "^1.0.1", + "ext": "^1.1.2" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.8.0.tgz", + "integrity": "sha512-K+Iayyo2LtyYhDSYwz5D5QdWw0hCacNzyq1Y821Xna2xSJj7cijoLLYmLxTQgcgZ9mC61nryMy9S7GRbYpI5Ig==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.10.0", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.14", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.3", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + } + }, + "node_modules/eslint-config-react-app": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/eslint-config-react-app/-/eslint-config-react-app-5.2.1.tgz", + "integrity": "sha512-pGIZ8t0mFLcV+6ZirRgYK6RVqUIKRIi9MmgzUEmrIknsn3AdO0I32asO86dJgloHq+9ZPl8UIg8mYrvgP5u2wQ==", + "dependencies": { + "confusing-browser-globals": "^1.0.9" + } + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.3.tgz", + "integrity": "sha512-b8crLDo0M5RSe5YG8Pu2DYBj71tSB6OvXkfzwbJU2w7y8P4/yo0MyF8jU26IEuEuHF2K5/gcAJE3LhQGqBBbVg==", + "dependencies": { + "debug": "^2.6.9", + "resolve": "^1.13.1" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/eslint-loader": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/eslint-loader/-/eslint-loader-3.0.3.tgz", + "integrity": "sha512-+YRqB95PnNvxNp1HEjQmvf9KNvCin5HXYYseOXVC2U0KEcw4IkQ2IQEBG46j7+gW39bMzeu0GsUhVbBY3Votpw==", + "dependencies": { + "fs-extra": "^8.1.0", + "loader-fs-cache": "^1.0.2", + "loader-utils": "^1.2.3", + "object-hash": "^2.0.1", + "schema-utils": "^2.6.1" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/eslint-module-utils": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.5.2.tgz", + "integrity": "sha512-LGScZ/JSlqGKiT8OC+cYRxseMjyqt6QO54nl281CK93unD89ijSeRV6An8Ci/2nvWVKe8K/Tqdm75RQoIOCr+Q==", + "dependencies": { + "debug": "^2.6.9", + "pkg-dir": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-module-utils/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/eslint-module-utils/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-module-utils/node_modules/pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dependencies": { + "find-up": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-flowtype": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-flowtype/-/eslint-plugin-flowtype-4.6.0.tgz", + "integrity": "sha512-W5hLjpFfZyZsXfo5anlu7HM970JBDqbEshAJUkeczP6BFCIfJXuiIBQXyberLRtOStT0OGPF8efeTbxlHk4LpQ==", + "dependencies": { + "lodash": "^4.17.15" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.20.1.tgz", + "integrity": "sha512-qQHgFOTjguR+LnYRoToeZWT62XM55MBVXObHM6SKFd1VzDcX/vqT1kAz8ssqigh5eMj8qXcRoXXGZpPP6RfdCw==", + "dependencies": { + "array-includes": "^3.0.3", + "array.prototype.flat": "^1.2.1", + "contains-path": "^0.1.0", + "debug": "^2.6.9", + "doctrine": "1.5.0", + "eslint-import-resolver-node": "^0.3.2", + "eslint-module-utils": "^2.4.1", + "has": "^1.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.0", + "read-pkg-up": "^2.0.0", + "resolve": "^1.12.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/doctrine": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", + "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "dependencies": { + "esutils": "^2.0.2", + "isarray": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/eslint-plugin-import/node_modules/load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/eslint-plugin-import/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "dependencies": { + "pify": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "dependencies": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-import/node_modules/read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-jsx-a11y": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.2.3.tgz", + "integrity": "sha512-CawzfGt9w83tyuVekn0GDPU9ytYtxyxyFZ3aSWROmnRRFQFT2BiPJd7jvRdzNDi6oLWaS2asMeYSNMjWTV4eNg==", + "dependencies": { + "@babel/runtime": "^7.4.5", + "aria-query": "^3.0.0", + "array-includes": "^3.0.3", + "ast-types-flow": "^0.0.7", + "axobject-query": "^2.0.2", + "damerau-levenshtein": "^1.0.4", + "emoji-regex": "^7.0.2", + "has": "^1.0.3", + "jsx-ast-utils": "^2.2.1" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/eslint-plugin-jsx-a11y/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/eslint-plugin-prefer-arrow": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/eslint-plugin-prefer-arrow/-/eslint-plugin-prefer-arrow-1.1.7.tgz", + "integrity": "sha512-epsA4g804mRovlOHSbeO1xxW7REGeUjULRME9MJTJDOVscNIA01AkR66TP4cmHDfD+w72EQ9cPhf37MbZiFI2w==", + "dev": true + }, + "node_modules/eslint-plugin-react": { + "version": "7.19.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.19.0.tgz", + "integrity": "sha512-SPT8j72CGuAP+JFbT0sJHOB80TX/pu44gQ4vXH/cq+hQTiY2PuZ6IHkqXJV6x1b28GDdo1lbInjKUrrdUf0LOQ==", + "dependencies": { + "array-includes": "^3.1.1", + "doctrine": "^2.1.0", + "has": "^1.0.3", + "jsx-ast-utils": "^2.2.3", + "object.entries": "^1.1.1", + "object.fromentries": "^2.0.2", + "object.values": "^1.1.1", + "prop-types": "^15.7.2", + "resolve": "^1.15.1", + "semver": "^6.3.0", + "string.prototype.matchall": "^4.0.2", + "xregexp": "^4.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-1.7.0.tgz", + "integrity": "sha512-iXTCFcOmlWvw4+TOE8CLWj6yX1GwzT0Y6cUfHHZqWnSk144VmVIRcVGtUAzrLES7C798lmvnt02C7rxaOX1HNA==", + "engines": { + "node": ">=7" + } + }, + "node_modules/eslint-plugin-react/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-react/node_modules/resolve": { + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.19.0.tgz", + "integrity": "sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==", + "dependencies": { + "is-core-module": "^2.1.0", + "path-parse": "^1.0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/eslint/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/eslint/node_modules/eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint/node_modules/globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dependencies": { + "type-fest": "^0.8.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/eslint/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/import-fresh": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint/node_modules/regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "engines": { + "node": ">=6.5.0" + } + }, + "node_modules/eslint/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dependencies": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esquery/node_modules/estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "node_modules/events": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz", + "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg==", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/eventsource": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-1.0.7.tgz", + "integrity": "sha512-4Ln17+vVT0k8aWq+t/bF5arcS3EpT9gYtW66EPacdj/mAFevznsnyoHLPy2BA8gbIQeIHoPsvwmfBftfcG//BQ==", + "dependencies": { + "original": "^1.0.0" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "dependencies": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/exec-sh": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz", + "integrity": "sha512-sEFIkc61v75sWeOe72qyrqg2Qg0OuLESziUDk/O/z2qgS15y2gWVFrI6f2Qn/qw/0/NCfCEsmNA4zOjkwEZT1A==" + }, + "node_modules/execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dependencies": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", + "dependencies": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/expand-brackets/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/expect": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-24.9.0.tgz", + "integrity": "sha512-wvVAx8XIol3Z5m9zvZXiyZOQ+sRJqNTIm6sGjdWlaZIeupQGO3WbYI+15D/AmEwZywL6wtJkbAbJtzkOfBuR0Q==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-styles": "^3.2.0", + "jest-get-type": "^24.9.0", + "jest-matcher-utils": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-regex-util": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/expect/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/expect/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/expect/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/express": { + "version": "4.17.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", + "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "dependencies": { + "accepts": "~1.3.7", + "array-flatten": "1.1.1", + "body-parser": "1.19.0", + "content-disposition": "0.5.3", + "content-type": "~1.0.4", + "cookie": "0.4.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.1.2", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.5", + "qs": "6.7.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.1.2", + "send": "0.17.1", + "serve-static": "1.14.1", + "setprototypeof": "1.1.1", + "statuses": "~1.5.0", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "node_modules/express/node_modules/qs": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", + "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/ext": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", + "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", + "dependencies": { + "type": "^2.0.0" + } + }, + "node_modules/ext/node_modules/type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", + "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=", + "dependencies": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend-shallow/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dependencies": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "engines": [ + "node >=0.6.0" + ] + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-glob": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.7.tgz", + "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==", + "dependencies": { + "@mrmlnc/readdir-enhanced": "^2.2.1", + "@nodelib/fs.stat": "^1.1.2", + "glob-parent": "^3.1.0", + "is-glob": "^4.0.0", + "merge2": "^1.2.3", + "micromatch": "^3.1.10" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "dependencies": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent/node_modules/is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "dependencies": { + "is-extglob": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "node_modules/faye-websocket": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.10.0.tgz", + "integrity": "sha1-TkkvjQTftviQA1B/btvy1QHnxvQ=", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/fb-watchman": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.1.tgz", + "integrity": "sha512-DkPJKQeY6kKwmuMretBhr7G6Vodr7bFwDYTXIkfG1gjvNpaxBTQV3PbXg6bR1c1UP4jPOX0jHUbbHANL9vRjVg==", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/figgy-pudding": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==" + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dependencies": { + "flat-cache": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/file-loader": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-4.3.0.tgz", + "integrity": "sha512-aKrYPYjF1yG3oX0kWRrqrSMfgftm7oJW5M+m4owoldH5C51C0RkIwB++JbRvEW3IU6/ZG5n8UvEcdgwOt2UOWA==", + "dependencies": { + "loader-utils": "^1.2.3", + "schema-utils": "^2.5.0" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, + "node_modules/filesize": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/filesize/-/filesize-6.0.1.tgz", + "integrity": "sha512-u4AYWPgbI5GBhs6id1KdImZWn5yfyFrrQ8OWZdN7ZMfA8Bf4HcO0BGo9bmUIEV8yrp8I1xVfJ/dn90GtFNNJcg==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dependencies": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, + "node_modules/flatten": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/flatten/-/flatten-1.0.3.tgz", + "integrity": "sha512-dVsPA/UwQ8+2uoFe5GHtiBMu48dWLTdsuEd7CKGlZlD78r1TTWBvDuFaFGKCo/ZfEr95Uk56vZoX86OsHkUeIg==" + }, + "node_modules/flush-write-stream": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz", + "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==", + "dependencies": { + "inherits": "^2.0.3", + "readable-stream": "^2.3.6" + } + }, + "node_modules/flush-write-stream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/flush-write-stream/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4=", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "engines": { + "node": "*" + } + }, + "node_modules/fork-ts-checker-webpack-plugin": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-3.1.1.tgz", + "integrity": "sha512-DuVkPNrM12jR41KM2e+N+styka0EgLkTnXmNcXdgOM37vtGeY+oCBK/Jx0hzSeEU6memFCtWb4htrHPMDfwwUQ==", + "dependencies": { + "babel-code-frame": "^6.22.0", + "chalk": "^2.4.1", + "chokidar": "^3.3.0", + "micromatch": "^3.1.10", + "minimatch": "^3.0.4", + "semver": "^5.6.0", + "tapable": "^1.0.0", + "worker-rpc": "^0.1.0" + }, + "engines": { + "node": ">=6.11.5", + "yarn": ">=1.0.0" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/fork-ts-checker-webpack-plugin/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/forwarded": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=", + "dependencies": { + "map-cache": "^0.2.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "dependencies": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + } + }, + "node_modules/from2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/from2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-write-stream-atomic": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", + "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=", + "dependencies": { + "graceful-fs": "^4.1.2", + "iferr": "^0.1.5", + "imurmurhash": "^0.1.4", + "readable-stream": "1 || 2" + } + }, + "node_modules/fs-write-stream-atomic/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/fs-write-stream-atomic/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "node_modules/fsevents": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", + "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=" + }, + "node_modules/gensync": { + "version": "1.0.0-beta.1", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz", + "integrity": "sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", + "integrity": "sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-own-enumerable-property-symbols": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", + "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==" + }, + "node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", + "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=" + }, + "node_modules/global-modules": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz", + "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", + "dependencies": { + "global-prefix": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "dependencies": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/global-prefix/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/globby": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-8.0.2.tgz", + "integrity": "sha512-yTzMmKygLp8RUpG1Ymu2VXPSJQZjNAZPD4ywgYEaG7e4tBJeUQBO8OpXrf1RCNcEs5alsoJYPAMiIHP0cmeC7w==", + "dependencies": { + "array-union": "^1.0.1", + "dir-glob": "2.0.0", + "fast-glob": "^2.0.2", + "glob": "^7.1.2", + "ignore": "^3.3.5", + "pify": "^3.0.0", + "slash": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/globby/node_modules/ignore": { + "version": "3.3.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==" + }, + "node_modules/globby/node_modules/slash": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", + "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" + }, + "node_modules/growly": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz", + "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=" + }, + "node_modules/gud": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/gud/-/gud-1.0.0.tgz", + "integrity": "sha512-zGEOVKFM5sVPPrYs7J5/hYEw2Pof8KCyOwyhG8sAF26mCAeUFAcYPu1mwB7hhpIP29zOIBaDqwuHdLp0jvZXjw==" + }, + "node_modules/gzip-size": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-5.1.1.tgz", + "integrity": "sha512-FNHi6mmoHvs1mxZAds4PpdCS6QG8B4C1krxJsMutgxl5t3+GlRTzzI3NEkifXx2pVsOvJdOGSmIgDhQ55FwdPA==", + "dependencies": { + "duplexer": "^0.1.1", + "pify": "^4.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/gzip-size/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==" + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "engines": { + "node": ">=4" + } + }, + "node_modules/har-validator": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", + "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", + "dependencies": { + "ajv": "^6.12.3", + "har-schema": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/harmony-reflect": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/harmony-reflect/-/harmony-reflect-1.6.1.tgz", + "integrity": "sha512-WJTeyp0JzGtHcuMsi7rw2VwtkvLa+JyfEKJCFyfcS0+CDkjQ5lHPu7zEhFZP+PDSRrEgXa5Ah0l1MbgbE41XjA==" + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-ansi/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=", + "dependencies": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=", + "dependencies": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/hash-base/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hex-color-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/hex-color-regex/-/hex-color-regex-1.1.0.tgz", + "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==" + }, + "node_modules/history": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/history/-/history-4.10.1.tgz", + "integrity": "sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==", + "dependencies": { + "@babel/runtime": "^7.1.2", + "loose-envify": "^1.2.0", + "resolve-pathname": "^3.0.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0", + "value-equal": "^1.0.1" + } + }, + "node_modules/hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "dependencies": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", + "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==" + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=", + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hsl-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz", + "integrity": "sha1-1JMwx4ntgZ4nakwNJy3/owsY/m4=" + }, + "node_modules/hsla-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hsla-regex/-/hsla-regex-1.0.0.tgz", + "integrity": "sha1-wc56MWjIxmFAM6S194d/OyJfnDg=" + }, + "node_modules/html-comment-regex": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.2.tgz", + "integrity": "sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ==" + }, + "node_modules/html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "dependencies": { + "whatwg-encoding": "^1.0.1" + } + }, + "node_modules/html-entities": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.3.1.tgz", + "integrity": "sha512-rhE/4Z3hIhzHAUKbW8jVcCyuT5oJCXXqhN/6mXXVCpzTmvJnoH2HL/bt3EZ6p55jbFJBeAe1ZNpL5BugLujxNA==" + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==" + }, + "node_modules/html-minifier-terser": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-5.1.1.tgz", + "integrity": "sha512-ZPr5MNObqnV/T9akshPKbVgyOqLmy+Bxo7juKCfTfnjNniTAMdy4hz21YQqoofMBJD2kdREaqPPdThoR78Tgxg==", + "dependencies": { + "camel-case": "^4.1.1", + "clean-css": "^4.2.3", + "commander": "^4.1.1", + "he": "^1.2.0", + "param-case": "^3.0.3", + "relateurl": "^0.2.7", + "terser": "^4.6.3" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/html-minifier-terser/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/html-webpack-plugin": { + "version": "4.0.0-beta.11", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-4.0.0-beta.11.tgz", + "integrity": "sha512-4Xzepf0qWxf8CGg7/WQM5qBB2Lc/NFI7MhU59eUDTkuQp3skZczH4UA1d6oQyDEIoMDgERVhRyTdtUPZ5s5HBg==", + "dependencies": { + "html-minifier-terser": "^5.0.1", + "loader-utils": "^1.2.3", + "lodash": "^4.17.15", + "pretty-error": "^2.1.1", + "tapable": "^1.1.3", + "util.promisify": "1.0.0" + }, + "engines": { + "node": ">=6.9" + } + }, + "node_modules/htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dependencies": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc=" + }, + "node_modules/http-errors": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", + "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.1", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/http-errors/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.19.1.tgz", + "integrity": "sha512-yHYTgWMQO8VvwNS22eLLloAkvungsKdKTLO8AJlftYIKNfJr3GK3zK0ZCfzDDGUBttdGc8xFy1mCitvNKQtC3Q==", + "dependencies": { + "http-proxy": "^1.17.0", + "is-glob": "^4.0.0", + "lodash": "^4.17.11", + "micromatch": "^3.1.10" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + }, + "engines": { + "node": ">=0.8", + "npm": ">=1.3.7" + } + }, + "node_modules/https-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", + "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" + }, + "node_modules/hyphenate-style-name": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.4.tgz", + "integrity": "sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==" + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/icss-utils": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-4.1.1.tgz", + "integrity": "sha512-4aFq7wvWyMHKgxsH8QQtGpvbASCf+eM3wPRLI6R+MgAnTCZ6STYsRvttLvRWK0Nfif5piF394St3HeJDaljGPA==", + "dependencies": { + "postcss": "^7.0.14" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/identity-obj-proxy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz", + "integrity": "sha1-lNK9qWCERT7zb7xarsN+D3nx/BQ=", + "dependencies": { + "harmony-reflect": "^1.4.6" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/iferr": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", + "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=" + }, + "node_modules/ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/immer": { + "version": "6.0.9", + "resolved": "https://registry.npmjs.org/immer/-/immer-6.0.9.tgz", + "integrity": "sha512-SyCYnAuiRf67Lvk0VkwFvwtDoEiCMjeamnHvRfnVDyc7re1/rQrNxuL+jJ7lA3WvdC4uznrvbmm+clJ9+XXatg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, + "node_modules/import-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-cwd/-/import-cwd-2.1.0.tgz", + "integrity": "sha1-qmzzbnInYShcs3HsZRn1PiQ1sKk=", + "dependencies": { + "import-from": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-2.0.0.tgz", + "integrity": "sha1-2BNVwVYS04bGH53dOSLUMEgipUY=", + "dependencies": { + "caller-path": "^2.0.0", + "resolve-from": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-from": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-from/-/import-from-2.1.0.tgz", + "integrity": "sha1-M1238qev/VOqpHHUuAId7ja387E=", + "dependencies": { + "resolve-from": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-local": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-2.0.0.tgz", + "integrity": "sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==", + "dependencies": { + "pkg-dir": "^3.0.0", + "resolve-cwd": "^2.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/indexes-of": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz", + "integrity": "sha1-8w9xbI4r00bHtn0985FVZqfAVgc=" + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==" + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "engines": { + "node": "*" + } + }, + "node_modules/inquirer": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.0.4.tgz", + "integrity": "sha512-Bu5Td5+j11sCkqfqmUTiwv+tWisMtP0L7Q8WrqA2C/BbBhy1YTdFrvjjlrKq8oagA/tLQBski2Gcx/Sqyi2qSQ==", + "dependencies": { + "ansi-escapes": "^4.2.1", + "chalk": "^2.4.2", + "cli-cursor": "^3.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.15", + "mute-stream": "0.0.8", + "run-async": "^2.2.0", + "rxjs": "^6.5.3", + "string-width": "^4.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/inquirer/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/inquirer/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/inquirer/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/internal-ip": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-4.3.0.tgz", + "integrity": "sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==", + "dependencies": { + "default-gateway": "^4.2.0", + "ipaddr.js": "^1.9.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/internal-slot": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.2.tgz", + "integrity": "sha512-2cQNfwhAfJIkU4KZPkDI+Gj5yNNnbqi40W9Gge6dfnk4TocEVm00B3bdiL+JINrbGJil2TeHvM4rETGzk/f/0g==", + "dependencies": { + "es-abstract": "^1.17.0-next.1", + "has": "^1.0.3", + "side-channel": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dependencies": { + "loose-envify": "^1.0.0" + } + }, + "node_modules/ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, + "node_modules/ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=", + "engines": { + "node": ">=4" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-absolute-url": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.1.0.tgz", + "integrity": "sha1-UFMN+4T8yap9vnhS6Do3uTufKqY=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, + "node_modules/is-callable": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", + "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dependencies": { + "ci-info": "^2.0.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/is-color-stop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-color-stop/-/is-color-stop-1.1.0.tgz", + "integrity": "sha1-z/9HGu5N1cnhWFmPvhKWe1za00U=", + "dependencies": { + "css-color-names": "^0.0.4", + "hex-color-regex": "^1.1.0", + "hsl-regex": "^1.0.0", + "hsla-regex": "^1.0.0", + "rgb-regex": "^1.0.1", + "rgba-regex": "^1.0.0" + } + }, + "node_modules/is-core-module": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.2.0.tgz", + "integrity": "sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ==", + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-date-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-descriptor/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-docker": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.1.1.tgz", + "integrity": "sha512-ZOoqiXfEwtGknTiuDEy8pN2CfE3TxMHprvNer1mXiqwkOT77Rw3YVrUQ52EqAOU3QAWDQ+bQdx7HJzrv7LS2Hw==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-in-browser": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-in-browser/-/is-in-browser-1.1.3.tgz", + "integrity": "sha1-Vv9NtoOgeMYILrldrX3GLh0E+DU=" + }, + "node_modules/is-negative-zero": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", + "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "dependencies": { + "is-path-inside": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "dependencies": { + "path-is-inside": "^1.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-regex": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", + "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", + "dependencies": { + "has-symbols": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", + "integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-resolvable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.1.0.tgz", + "integrity": "sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg==" + }, + "node_modules/is-root": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-root/-/is-root-2.1.0.tgz", + "integrity": "sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-string": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", + "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-svg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-svg/-/is-svg-3.0.0.tgz", + "integrity": "sha512-gi4iHK53LR2ujhLVVj+37Ykh9GLqYHX6JOVXbLAucaG/Cqw9xwdFOjDM2qeifLs1sF1npXXFvDu0r5HNgCMrzQ==", + "dependencies": { + "html-comment-regex": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "dependencies": { + "has-symbols": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-wsl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", + "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "node_modules/istanbul-lib-coverage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", + "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz", + "integrity": "sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA==", + "dependencies": { + "@babel/generator": "^7.4.0", + "@babel/parser": "^7.4.3", + "@babel/template": "^7.4.0", + "@babel/traverse": "^7.4.3", + "@babel/types": "^7.4.0", + "istanbul-lib-coverage": "^2.0.5", + "semver": "^6.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-report": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz", + "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==", + "dependencies": { + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-report/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.6.tgz", + "integrity": "sha512-R47KzMtDJH6X4/YW9XTx+jrLnZnscW4VpNN+1PViSYTejLVPWv7oov+Duf8YQSPyVRUvueQqz1TcsC6mooZTXw==", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "rimraf": "^2.6.3", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-reports": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.7.tgz", + "integrity": "sha512-uu1F/L1o5Y6LzPVSVZXNOoD/KXpJue9aeLRd0sM9uMXfZvzomB0WxVamWb5ue8kA2vVWEmW7EG+A5n3f1kqHKg==", + "dependencies": { + "html-escaper": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jest": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-24.9.0.tgz", + "integrity": "sha512-YvkBL1Zm7d2B1+h5fHEOdyjCG+sGMz4f8D86/0HiqJ6MB4MnDc8FgP5vdWsGnemOQro7lnYo8UakZ3+5A0jxGw==", + "dependencies": { + "import-local": "^2.0.0", + "jest-cli": "^24.9.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-changed-files": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-24.9.0.tgz", + "integrity": "sha512-6aTWpe2mHF0DhL28WjdkO8LyGjs3zItPET4bMSeXU6T3ub4FPMw+mcOcbdGXQOAfmLcxofD23/5Bl9Z4AkFwqg==", + "dependencies": { + "@jest/types": "^24.9.0", + "execa": "^1.0.0", + "throat": "^4.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-config": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-24.9.0.tgz", + "integrity": "sha512-RATtQJtVYQrp7fvWg6f5y3pEFj9I+H8sWw4aKxnDZ96mob5i5SD6ZEGWgMLXQ4LE8UurrjbdlLWdUeo+28QpfQ==", + "dependencies": { + "@babel/core": "^7.1.0", + "@jest/test-sequencer": "^24.9.0", + "@jest/types": "^24.9.0", + "babel-jest": "^24.9.0", + "chalk": "^2.0.1", + "glob": "^7.1.1", + "jest-environment-jsdom": "^24.9.0", + "jest-environment-node": "^24.9.0", + "jest-get-type": "^24.9.0", + "jest-jasmine2": "^24.9.0", + "jest-regex-util": "^24.3.0", + "jest-resolve": "^24.9.0", + "jest-util": "^24.9.0", + "jest-validate": "^24.9.0", + "micromatch": "^3.1.10", + "pretty-format": "^24.9.0", + "realpath-native": "^1.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-config/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-config/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-config/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-config/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-config/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-config/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-config/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-config/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-diff": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-25.5.0.tgz", + "integrity": "sha512-z1kygetuPiREYdNIumRpAHY6RXiGmp70YHptjdaxTWGmA085W3iCnXNx0DhflK3vwrKmrRWyY1wUpkPMVxMK7A==", + "dependencies": { + "chalk": "^3.0.0", + "diff-sequences": "^25.2.6", + "jest-get-type": "^25.2.6", + "pretty-format": "^25.5.0" + }, + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/jest-diff/node_modules/jest-get-type": { + "version": "25.2.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-25.2.6.tgz", + "integrity": "sha512-DxjtyzOHjObRM+sM1knti6or+eOgcGU4xVSb2HNP1TqO4ahsT+rqZg+nyqHWJSvWgKC5cG3QjGFBqxLghiF/Ig==", + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/jest-docblock": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-24.9.0.tgz", + "integrity": "sha512-F1DjdpDMJMA1cN6He0FNYNZlo3yYmOtRUnktrT9Q37njYzC5WEaDdmbynIgy0L/IvXvvgsG8OsqhLPXTpfmZAA==", + "dependencies": { + "detect-newline": "^2.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-each": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-24.9.0.tgz", + "integrity": "sha512-ONi0R4BvW45cw8s2Lrx8YgbeXL1oCQ/wIDwmsM3CqM/nlblNCPmnC3IPQlMbRFZu3wKdQ2U8BqM6lh3LJ5Bsog==", + "dependencies": { + "@jest/types": "^24.9.0", + "chalk": "^2.0.1", + "jest-get-type": "^24.9.0", + "jest-util": "^24.9.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-each/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-each/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-each/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-each/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-each/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-each/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-each/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-each/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-environment-jsdom": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-24.9.0.tgz", + "integrity": "sha512-Zv9FV9NBRzLuALXjvRijO2351DRQeLYXtpD4xNvfoVFw21IOKNhZAEUKcbiEtjTkm2GsJ3boMVgkaR7rN8qetA==", + "dependencies": { + "@jest/environment": "^24.9.0", + "@jest/fake-timers": "^24.9.0", + "@jest/types": "^24.9.0", + "jest-mock": "^24.9.0", + "jest-util": "^24.9.0", + "jsdom": "^11.5.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-environment-jsdom-fourteen": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/jest-environment-jsdom-fourteen/-/jest-environment-jsdom-fourteen-1.0.1.tgz", + "integrity": "sha512-DojMX1sY+at5Ep+O9yME34CdidZnO3/zfPh8UW+918C5fIZET5vCjfkegixmsi7AtdYfkr4bPlIzmWnlvQkP7Q==", + "dependencies": { + "@jest/environment": "^24.3.0", + "@jest/fake-timers": "^24.3.0", + "@jest/types": "^24.3.0", + "jest-mock": "^24.0.0", + "jest-util": "^24.0.0", + "jsdom": "^14.1.0" + } + }, + "node_modules/jest-environment-jsdom-fourteen/node_modules/acorn": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/jest-environment-jsdom-fourteen/node_modules/jsdom": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-14.1.0.tgz", + "integrity": "sha512-O901mfJSuTdwU2w3Sn+74T+RnDVP+FuV5fH8tcPWyqrseRAb0s5xOtPgCFiPOtLcyK7CLIJwPyD83ZqQWvA5ng==", + "dependencies": { + "abab": "^2.0.0", + "acorn": "^6.0.4", + "acorn-globals": "^4.3.0", + "array-equal": "^1.0.0", + "cssom": "^0.3.4", + "cssstyle": "^1.1.1", + "data-urls": "^1.1.0", + "domexception": "^1.0.1", + "escodegen": "^1.11.0", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.1.3", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.5", + "saxes": "^3.1.9", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.5.0", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.1.2", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^6.1.2", + "xml-name-validator": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-environment-jsdom-fourteen/node_modules/parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" + }, + "node_modules/jest-environment-jsdom-fourteen/node_modules/whatwg-url": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.1.0.tgz", + "integrity": "sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/jest-environment-jsdom-fourteen/node_modules/ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-24.9.0.tgz", + "integrity": "sha512-6d4V2f4nxzIzwendo27Tr0aFm+IXWa0XEUnaH6nU0FMaozxovt+sfRvh4J47wL1OvF83I3SSTu0XK+i4Bqe7uA==", + "dependencies": { + "@jest/environment": "^24.9.0", + "@jest/fake-timers": "^24.9.0", + "@jest/types": "^24.9.0", + "jest-mock": "^24.9.0", + "jest-util": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-get-type": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", + "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-haste-map": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-24.9.0.tgz", + "integrity": "sha512-kfVFmsuWui2Sj1Rp1AJ4D9HqJwE4uwTlS/vO+eRUaMmd54BFpli2XhMQnPC2k4cHFVbB2Q2C+jtI1AGLgEnCjQ==", + "dependencies": { + "@jest/types": "^24.9.0", + "anymatch": "^2.0.0", + "fb-watchman": "^2.0.0", + "fsevents": "^1.2.7", + "graceful-fs": "^4.1.15", + "invariant": "^2.2.4", + "jest-serializer": "^24.9.0", + "jest-util": "^24.9.0", + "jest-worker": "^24.9.0", + "micromatch": "^3.1.10", + "sane": "^4.0.3", + "walker": "^1.0.7" + }, + "engines": { + "node": ">= 6" + }, + "optionalDependencies": { + "fsevents": "^1.2.7" + } + }, + "node_modules/jest-haste-map/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, + "node_modules/jest-jasmine2": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-24.9.0.tgz", + "integrity": "sha512-Cq7vkAgaYKp+PsX+2/JbTarrk0DmNhsEtqBXNwUHkdlbrTBLtMJINADf2mf5FkowNsq8evbPc07/qFO0AdKTzw==", + "dependencies": { + "@babel/traverse": "^7.1.0", + "@jest/environment": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "chalk": "^2.0.1", + "co": "^4.6.0", + "expect": "^24.9.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^24.9.0", + "jest-matcher-utils": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-runtime": "^24.9.0", + "jest-snapshot": "^24.9.0", + "jest-util": "^24.9.0", + "pretty-format": "^24.9.0", + "throat": "^4.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-jasmine2/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-jasmine2/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-jasmine2/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-jasmine2/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-jasmine2/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-jasmine2/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-jasmine2/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-jasmine2/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-leak-detector": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-24.9.0.tgz", + "integrity": "sha512-tYkFIDsiKTGwb2FG1w8hX9V0aUb2ot8zY/2nFg087dUageonw1zrLMP4W6zsRO59dPkTSKie+D4rhMuP9nRmrA==", + "dependencies": { + "jest-get-type": "^24.9.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-leak-detector/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-leak-detector/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-leak-detector/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-leak-detector/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-leak-detector/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-matcher-utils": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-24.9.0.tgz", + "integrity": "sha512-OZz2IXsu6eaiMAwe67c1T+5tUAtQyQx27/EMEkbFAGiw52tB9em+uGbzpcgYVpA8wl0hlxKPZxrly4CXU/GjHA==", + "dependencies": { + "chalk": "^2.0.1", + "jest-diff": "^24.9.0", + "jest-get-type": "^24.9.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-matcher-utils/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-matcher-utils/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-matcher-utils/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-matcher-utils/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-matcher-utils/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-matcher-utils/node_modules/diff-sequences": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz", + "integrity": "sha512-Dj6Wk3tWyTE+Fo1rW8v0Xhwk80um6yFYKbuAxc9c3EZxIHFDYwbi34Uk42u1CdnIiVorvt4RmlSDjIPyzGC2ew==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-matcher-utils/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-matcher-utils/node_modules/jest-diff": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-24.9.0.tgz", + "integrity": "sha512-qMfrTs8AdJE2iqrTp0hzh7kTd2PQWrsFyj9tORoKmu32xjPjeE4NyjVRDz8ybYwqS2ik8N4hsIpiVTyFeo2lBQ==", + "dependencies": { + "chalk": "^2.0.1", + "diff-sequences": "^24.9.0", + "jest-get-type": "^24.9.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-matcher-utils/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-matcher-utils/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-message-util": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-24.9.0.tgz", + "integrity": "sha512-oCj8FiZ3U0hTP4aSui87P4L4jC37BtQwUMqk+zk/b11FR19BJDeZsZAvIHutWnmtw7r85UmR3CEWZ0HWU2mAlw==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "@types/stack-utils": "^1.0.1", + "chalk": "^2.0.1", + "micromatch": "^3.1.10", + "slash": "^2.0.0", + "stack-utils": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-message-util/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-message-util/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-message-util/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-message-util/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-message-util/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-message-util/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-mock": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-24.9.0.tgz", + "integrity": "sha512-3BEYN5WbSq9wd+SyLDES7AHnjH9A/ROBwmz7l2y+ol+NtSFO8DYiEBzoO1CeFc9a8DYy10EO4dDFVv/wN3zl1w==", + "dependencies": { + "@jest/types": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz", + "integrity": "sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-regex-util": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-24.9.0.tgz", + "integrity": "sha512-05Cmb6CuxaA+Ys6fjr3PhvV3bGQmO+2p2La4hFbU+W5uOc479f7FdLXUWXw4pYMAhhSZIuKHwSXSu6CsSBAXQA==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-resolve": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-24.9.0.tgz", + "integrity": "sha512-TaLeLVL1l08YFZAt3zaPtjiVvyy4oSA6CRe+0AFPPVX3Q/VI0giIWWoAvoS5L96vj9Dqxj4fB5p2qrHCmTU/MQ==", + "dependencies": { + "@jest/types": "^24.9.0", + "browser-resolve": "^1.11.3", + "chalk": "^2.0.1", + "jest-pnp-resolver": "^1.2.1", + "realpath-native": "^1.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-24.9.0.tgz", + "integrity": "sha512-Fm7b6AlWnYhT0BXy4hXpactHIqER7erNgIsIozDXWl5dVm+k8XdGVe1oTg1JyaFnOxarMEbax3wyRJqGP2Pq+g==", + "dependencies": { + "@jest/types": "^24.9.0", + "jest-regex-util": "^24.3.0", + "jest-snapshot": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-resolve/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-resolve/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-resolve/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-resolve/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-resolve/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-resolve/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runner": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-24.9.0.tgz", + "integrity": "sha512-KksJQyI3/0mhcfspnxxEOBueGrd5E4vV7ADQLT9ESaCzz02WnbdbKWIf5Mkaucoaj7obQckYPVX6JJhgUcoWWg==", + "dependencies": { + "@jest/console": "^24.7.1", + "@jest/environment": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "chalk": "^2.4.2", + "exit": "^0.1.2", + "graceful-fs": "^4.1.15", + "jest-config": "^24.9.0", + "jest-docblock": "^24.3.0", + "jest-haste-map": "^24.9.0", + "jest-jasmine2": "^24.9.0", + "jest-leak-detector": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-resolve": "^24.9.0", + "jest-runtime": "^24.9.0", + "jest-util": "^24.9.0", + "jest-worker": "^24.6.0", + "source-map-support": "^0.5.6", + "throat": "^4.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-runner/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runner/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runner/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-runner/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-runner/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runner/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runtime": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-24.9.0.tgz", + "integrity": "sha512-8oNqgnmF3v2J6PVRM2Jfuj8oX3syKmaynlDMMKQ4iyzbQzIG6th5ub/lM2bCMTmoTKM3ykcUYI2Pw9xwNtjMnw==", + "dependencies": { + "@jest/console": "^24.7.1", + "@jest/environment": "^24.9.0", + "@jest/source-map": "^24.3.0", + "@jest/transform": "^24.9.0", + "@jest/types": "^24.9.0", + "@types/yargs": "^13.0.0", + "chalk": "^2.0.1", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.1.15", + "jest-config": "^24.9.0", + "jest-haste-map": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-mock": "^24.9.0", + "jest-regex-util": "^24.3.0", + "jest-resolve": "^24.9.0", + "jest-snapshot": "^24.9.0", + "jest-util": "^24.9.0", + "jest-validate": "^24.9.0", + "realpath-native": "^1.1.0", + "slash": "^2.0.0", + "strip-bom": "^3.0.0", + "yargs": "^13.3.0" + }, + "bin": { + "jest-runtime": "bin/jest-runtime.js" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-runtime/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runtime/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runtime/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-runtime/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-runtime/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-runtime/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-serializer": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-serializer/-/jest-serializer-24.9.0.tgz", + "integrity": "sha512-DxYipDr8OvfrKH3Kel6NdED3OXxjvxXZ1uIY2I9OFbGg+vUkkg7AGvi65qbhbWNPvDckXmzMPbK3u3HaDO49bQ==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-snapshot": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-24.9.0.tgz", + "integrity": "sha512-uI/rszGSs73xCM0l+up7O7a40o90cnrk429LOiK3aeTvfC0HHmldbd81/B7Ix81KSFe1lwkbl7GnBGG4UfuDew==", + "dependencies": { + "@babel/types": "^7.0.0", + "@jest/types": "^24.9.0", + "chalk": "^2.0.1", + "expect": "^24.9.0", + "jest-diff": "^24.9.0", + "jest-get-type": "^24.9.0", + "jest-matcher-utils": "^24.9.0", + "jest-message-util": "^24.9.0", + "jest-resolve": "^24.9.0", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "pretty-format": "^24.9.0", + "semver": "^6.2.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-snapshot/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-snapshot/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-snapshot/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-snapshot/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-snapshot/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-snapshot/node_modules/diff-sequences": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-24.9.0.tgz", + "integrity": "sha512-Dj6Wk3tWyTE+Fo1rW8v0Xhwk80um6yFYKbuAxc9c3EZxIHFDYwbi34Uk42u1CdnIiVorvt4RmlSDjIPyzGC2ew==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-snapshot/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-snapshot/node_modules/jest-diff": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-24.9.0.tgz", + "integrity": "sha512-qMfrTs8AdJE2iqrTp0hzh7kTd2PQWrsFyj9tORoKmu32xjPjeE4NyjVRDz8ybYwqS2ik8N4hsIpiVTyFeo2lBQ==", + "dependencies": { + "chalk": "^2.0.1", + "diff-sequences": "^24.9.0", + "jest-get-type": "^24.9.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-snapshot/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-snapshot/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-util": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-24.9.0.tgz", + "integrity": "sha512-x+cZU8VRmOJxbA1K5oDBdxQmdq0OIdADarLxk0Mq+3XS4jgvhG/oKGWcIDCtPG0HgjxOYvF+ilPJQsAyXfbNOg==", + "dependencies": { + "@jest/console": "^24.9.0", + "@jest/fake-timers": "^24.9.0", + "@jest/source-map": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "callsites": "^3.0.0", + "chalk": "^2.0.1", + "graceful-fs": "^4.1.15", + "is-ci": "^2.0.0", + "mkdirp": "^0.5.1", + "slash": "^2.0.0", + "source-map": "^0.6.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-util/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-util/node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-util/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-util/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-util/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-util/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-util/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jest-util/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-validate": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-24.9.0.tgz", + "integrity": "sha512-HPIt6C5ACwiqSiwi+OfSSHbK8sG7akG8eATl+IPKaeIjtPOeBUd/g3J7DghugzxrGjI93qS/+RPKe1H6PqvhRQ==", + "dependencies": { + "@jest/types": "^24.9.0", + "camelcase": "^5.3.1", + "chalk": "^2.0.1", + "jest-get-type": "^24.9.0", + "leven": "^3.1.0", + "pretty-format": "^24.9.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-validate/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/jest-validate/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-validate/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-validate/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-validate/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-validate/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-validate/node_modules/pretty-format": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", + "integrity": "sha512-00ZMZUiHaJrNfk33guavqgvfJS30sLYf0f8+Srklv0AMPodGGHcoHgksZ3OThYnIvOd+8yMCn0YiEOogjlgsnA==", + "dependencies": { + "@jest/types": "^24.9.0", + "ansi-regex": "^4.0.0", + "ansi-styles": "^3.2.0", + "react-is": "^16.8.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-validate/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watch-typeahead": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/jest-watch-typeahead/-/jest-watch-typeahead-0.4.2.tgz", + "integrity": "sha512-f7VpLebTdaXs81rg/oj4Vg/ObZy2QtGzAmGLNsqUS5G5KtSN68tFcIsbvNODfNyQxU78g7D8x77o3bgfBTR+2Q==", + "dependencies": { + "ansi-escapes": "^4.2.1", + "chalk": "^2.4.1", + "jest-regex-util": "^24.9.0", + "jest-watcher": "^24.3.0", + "slash": "^3.0.0", + "string-length": "^3.1.0", + "strip-ansi": "^5.0.0" + } + }, + "node_modules/jest-watch-typeahead/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watch-typeahead/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watch-typeahead/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-watch-typeahead/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-watch-typeahead/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watch-typeahead/node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-watch-typeahead/node_modules/string-length": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-3.1.0.tgz", + "integrity": "sha512-Ttp5YvkGm5v9Ijagtaz1BnN+k9ObpvS0eIBblPMp2YWL8FBmi9qblQ9fexc2k/CXFgrTIteU3jAw3payCnwSTA==", + "dependencies": { + "astral-regex": "^1.0.0", + "strip-ansi": "^5.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-watch-typeahead/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watcher": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-24.9.0.tgz", + "integrity": "sha512-+/fLOfKPXXYJDYlks62/4R4GoT+GU1tYZed99JSCOsmzkkF7727RqKrjNAxtfO4YpGv11wybgRvCjR73lK2GZw==", + "dependencies": { + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "@types/yargs": "^13.0.0", + "ansi-escapes": "^3.0.0", + "chalk": "^2.0.1", + "jest-util": "^24.9.0", + "string-length": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-watcher/node_modules/ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watcher/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watcher/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watcher/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest-watcher/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest-watcher/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-watcher/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-worker": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-24.9.0.tgz", + "integrity": "sha512-51PE4haMSXcHohnSMdM42anbvZANYTqMrr52tVKPqqsPJMzoP6FYYDVqahX/HrAoKEKz3uUPzSvKs9A3qR4iVw==", + "dependencies": { + "merge-stream": "^2.0.0", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest-worker/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jest/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jest/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jest/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/jest/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/jest/node_modules/jest-cli": { + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-24.9.0.tgz", + "integrity": "sha512-+VLRKyitT3BWoMeSUIHRxV/2g8y9gw91Jh5z2UmXZzkZKpbC08CSehVxgHUwTpy+HwGcns/tqafQDJW7imYvGg==", + "dependencies": { + "@jest/core": "^24.9.0", + "@jest/test-result": "^24.9.0", + "@jest/types": "^24.9.0", + "chalk": "^2.0.1", + "exit": "^0.1.2", + "import-local": "^2.0.0", + "is-ci": "^2.0.0", + "jest-config": "^24.9.0", + "jest-util": "^24.9.0", + "jest-validate": "^24.9.0", + "prompts": "^2.0.1", + "realpath-native": "^1.1.0", + "yargs": "^13.3.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/jest/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "node_modules/jsdom": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-11.12.0.tgz", + "integrity": "sha512-y8Px43oyiBM13Zc1z780FrfNLJCXTL40EWlty/LXUtcjykRBNgLlCjWXpfSPBl2iv+N7koQN+dvqszHZgT/Fjw==", + "dependencies": { + "abab": "^2.0.0", + "acorn": "^5.5.3", + "acorn-globals": "^4.1.0", + "array-equal": "^1.0.0", + "cssom": ">= 0.3.2 < 0.4.0", + "cssstyle": "^1.0.0", + "data-urls": "^1.0.0", + "domexception": "^1.0.1", + "escodegen": "^1.9.1", + "html-encoding-sniffer": "^1.0.2", + "left-pad": "^1.3.0", + "nwsapi": "^2.0.7", + "parse5": "4.0.0", + "pn": "^1.1.0", + "request": "^2.87.0", + "request-promise-native": "^1.0.5", + "sax": "^1.2.4", + "symbol-tree": "^3.2.2", + "tough-cookie": "^2.3.4", + "w3c-hr-time": "^1.0.1", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.3", + "whatwg-mimetype": "^2.1.0", + "whatwg-url": "^6.4.1", + "ws": "^5.2.0", + "xml-name-validator": "^3.0.0" + } + }, + "node_modules/jsdom/node_modules/acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.0.tgz", + "integrity": "sha512-o3aP+RsWDJZayj1SbHNQAI8x0v3T3SKiGoZlNYfbUP1S3omJQ6i9CnqADqkSPaOAxwua4/1YWx5CM7oiChJt2Q==" + }, + "node_modules/json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "node_modules/json-stable-stringify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", + "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", + "dependencies": { + "jsonify": "~0.0.0" + } + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=" + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "node_modules/json3": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.3.tgz", + "integrity": "sha512-c7/8mbUsKigAbLkD5B010BK4D9LZm7A1pNItkEwiUZRpIN66exu/e7YQWysGun+TRKaJp8MhemM+VkfWv42aCA==" + }, + "node_modules/json5": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz", + "integrity": "sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA==", + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonify": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", + "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" + }, + "node_modules/jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "node_modules/jss": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss/-/jss-10.4.0.tgz", + "integrity": "sha512-l7EwdwhsDishXzqTc3lbsbyZ83tlUl5L/Hb16pHCvZliA9lRDdNBZmHzeJHP0sxqD0t1mrMmMR8XroR12JBYzw==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "csstype": "^3.0.2", + "is-in-browser": "^1.1.3", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/jss-plugin-camel-case": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.4.0.tgz", + "integrity": "sha512-9oDjsQ/AgdBbMyRjc06Kl3P8lDCSEts2vYZiPZfGAxbGCegqE4RnMob3mDaBby5H9vL9gWmyyImhLRWqIkRUCw==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "hyphenate-style-name": "^1.0.3", + "jss": "10.4.0" + } + }, + "node_modules/jss-plugin-default-unit": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-default-unit/-/jss-plugin-default-unit-10.4.0.tgz", + "integrity": "sha512-BYJ+Y3RUYiMEgmlcYMLqwbA49DcSWsGgHpVmEEllTC8MK5iJ7++pT9TnKkKBnNZZxTV75ycyFCR5xeLSOzVm4A==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.4.0" + } + }, + "node_modules/jss-plugin-global": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-global/-/jss-plugin-global-10.4.0.tgz", + "integrity": "sha512-b8IHMJUmv29cidt3nI4bUI1+Mo5RZE37kqthaFpmxf5K7r2aAegGliAw4hXvA70ca6ckAoXMUl4SN/zxiRcRag==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.4.0" + } + }, + "node_modules/jss-plugin-nested": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-nested/-/jss-plugin-nested-10.4.0.tgz", + "integrity": "sha512-cKgpeHIxAP0ygeWh+drpLbrxFiak6zzJ2toVRi/NmHbpkNaLjTLgePmOz5+67ln3qzJiPdXXJB1tbOyYKAP4Pw==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.4.0", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/jss-plugin-props-sort": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-props-sort/-/jss-plugin-props-sort-10.4.0.tgz", + "integrity": "sha512-j/t0R40/2fp+Nzt6GgHeUFnHVY2kPGF5drUVlgkcwYoHCgtBDOhTTsOfdaQFW6sHWfoQYgnGV4CXdjlPiRrzwA==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.4.0" + } + }, + "node_modules/jss-plugin-rule-value-function": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.4.0.tgz", + "integrity": "sha512-w8504Cdfu66+0SJoLkr6GUQlEb8keHg8ymtJXdVHWh0YvFxDG2l/nS93SI5Gfx0fV29dO6yUugXnKzDFJxrdFQ==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.4.0", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/jss-plugin-vendor-prefixer": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.4.0.tgz", + "integrity": "sha512-DpF+/a+GU8hMh/948sBGnKSNfKkoHg2p9aRFUmyoyxgKjOeH9n74Ht3Yt8lOgdZsuWNJbPrvaa3U4PXKwxVpTQ==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "css-vendor": "^2.0.8", + "jss": "10.4.0" + } + }, + "node_modules/jss/node_modules/csstype": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.5.tgz", + "integrity": "sha512-uVDi8LpBUKQj6sdxNaTetL6FpeCqTjOvAQuQUa/qAqq8oOd4ivkbhgnqayl0dnPal8Tb/yB1tF+gOvCBiicaiQ==" + }, + "node_modules/jsx-ast-utils": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-2.4.1.tgz", + "integrity": "sha512-z1xSldJ6imESSzOjd3NNkieVJKRlKYSOtMG8SFyCj2FIrvSaSuli/WjpBkEzCBoR9bYYYFgqJw61Xhu7Lcgk+w==", + "dependencies": { + "array-includes": "^3.1.1", + "object.assign": "^4.1.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/killable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/killable/-/killable-1.0.1.tgz", + "integrity": "sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==" + }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/last-call-webpack-plugin": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/last-call-webpack-plugin/-/last-call-webpack-plugin-3.0.0.tgz", + "integrity": "sha512-7KI2l2GIZa9p2spzPIVZBYyNKkN+e/SQPpnjlTiPhdbDW3F86tdKKELxKpzJ5sgU19wQWsACULZmpTPYHeWO5w==", + "dependencies": { + "lodash": "^4.17.5", + "webpack-sources": "^1.1.0" + } + }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/left-pad": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz", + "integrity": "sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==" + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/levenary": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/levenary/-/levenary-1.1.1.tgz", + "integrity": "sha512-mkAdOIt79FD6irqjYSs4rdbnlT5vRonMEvBVPVb3XmevfS8kgRXwfes0dhPdEtzTWD/1eNE/Bm/G1iRt6DcnQQ==", + "dependencies": { + "leven": "^3.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", + "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=" + }, + "node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/loader-fs-cache": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/loader-fs-cache/-/loader-fs-cache-1.0.3.tgz", + "integrity": "sha512-ldcgZpjNJj71n+2Mf6yetz+c9bM4xpKtNds4LbqXzU/PTdeAX0g3ytnU1AJMEcTk2Lex4Smpe3Q/eCTsvUBxbA==", + "dependencies": { + "find-cache-dir": "^0.1.1", + "mkdirp": "^0.5.1" + } + }, + "node_modules/loader-fs-cache/node_modules/find-cache-dir": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-0.1.1.tgz", + "integrity": "sha1-yN765XyKUqinhPnjHFfHQumToLk=", + "dependencies": { + "commondir": "^1.0.1", + "mkdirp": "^0.5.1", + "pkg-dir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loader-fs-cache/node_modules/find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dependencies": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loader-fs-cache/node_modules/path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dependencies": { + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loader-fs-cache/node_modules/pkg-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-1.0.0.tgz", + "integrity": "sha1-ektQio1bstYp1EcFb/TpyTFM89Q=", + "dependencies": { + "find-up": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loader-runner": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", + "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==", + "engines": { + "node": ">=4.3.0 <5.0.0 || >=5.10" + } + }, + "node_modules/loader-utils": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", + "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/loader-utils/node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "node_modules/lodash._reinterpolate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", + "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=" + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=" + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, + "node_modules/lodash.template": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", + "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==", + "dependencies": { + "lodash._reinterpolate": "^3.0.0", + "lodash.templatesettings": "^4.0.0" + } + }, + "node_modules/lodash.templatesettings": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz", + "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==", + "dependencies": { + "lodash._reinterpolate": "^3.0.0" + } + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, + "node_modules/loglevel": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.7.0.tgz", + "integrity": "sha512-i2sY04nal5jDcagM3FMfG++T69GEEM8CYuOfeOIvmXzOIcwE9a/CJPR0MFM97pYMj/u10lzz7/zd7+qwhrBTqQ==", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lower-case": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.1.tgz", + "integrity": "sha512-LiWgfDLLb1dwbFQZsSglpRj+1ctGnayXz3Uv0/WO8n558JycT5fg6zkNcnW0G68Nn0aEldTFeEfmjCfmqry/rQ==", + "dependencies": { + "tslib": "^1.10.0" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lru-cache/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + }, + "node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/makeerror": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.11.tgz", + "integrity": "sha1-4BpckQnyr3lmDk6LlYd5AYT1qWw=", + "dependencies": { + "tmpl": "1.0.x" + } + }, + "node_modules/mamacro": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/mamacro/-/mamacro-0.0.3.tgz", + "integrity": "sha512-qMEwh+UujcQ+kbz3T6V+wAmO2U8veoq2w+3wY8MquqwVA3jChfwY+Tk52GZKDfACEPjuZ7r2oJLejwpt8jtwTA==" + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=", + "dependencies": { + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/mdn-data": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", + "integrity": "sha512-iV3XNKw06j5Q7mi6h+9vbx23Tv7JkjEVgKHW4pimwyDGWm0OIQntJJ+u1C6mg6mK1EaTv42XQ7w76yuzH7M2cA==" + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memory-fs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", + "integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=", + "dependencies": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + }, + "node_modules/memory-fs/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/memory-fs/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/merge-deep": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz", + "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==", + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/microevent.ts": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/microevent.ts/-/microevent.ts-0.1.1.tgz", + "integrity": "sha512-jo1OfR4TaEwd5HOrt5+tAZ9mqT4jmpNAusXtyfNzqVm9uiSYFZlKM1wYL4oU7azZW/PxQW53wM0S6OR1JHNa2g==" + }, + "node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/micromatch/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "dependencies": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "bin": { + "miller-rabin": "bin/miller-rabin" + } + }, + "node_modules/miller-rabin/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/mime": { + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", + "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "dependencies": { + "mime-db": "1.44.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/mini-create-react-context": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/mini-create-react-context/-/mini-create-react-context-0.3.2.tgz", + "integrity": "sha512-2v+OeetEyliMt5VHMXsBhABoJ0/M4RCe7fatd/fBy6SMiKazUSEt3gxxypfnk2SHMkdBYvorHRoQxuGoiwbzAw==", + "dependencies": { + "@babel/runtime": "^7.4.0", + "gud": "^1.0.0", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/mini-css-extract-plugin": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-0.9.0.tgz", + "integrity": "sha512-lp3GeY7ygcgAmVIcRPBVhIkf8Us7FZjA+ILpal44qLdSu11wmjKQ3d9k15lfD7pO4esu9eUIAW7qiYIBppv40A==", + "dependencies": { + "loader-utils": "^1.1.0", + "normalize-url": "1.9.1", + "schema-utils": "^1.0.0", + "webpack-sources": "^1.1.0" + }, + "engines": { + "node": ">= 6.9.0" + } + }, + "node_modules/mini-css-extract-plugin/node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "node_modules/minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "node_modules/minipass": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz", + "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mississippi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", + "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "dependencies": { + "concat-stream": "^1.5.0", + "duplexify": "^3.4.2", + "end-of-stream": "^1.1.0", + "flush-write-stream": "^1.0.0", + "from2": "^2.1.0", + "parallel-transform": "^1.1.0", + "pump": "^3.0.0", + "pumpify": "^1.3.3", + "stream-each": "^1.1.0", + "through2": "^2.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dependencies": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-deep/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=", + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/move-concurrently": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", + "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=", + "dependencies": { + "aproba": "^1.1.1", + "copy-concurrently": "^1.0.0", + "fs-write-stream-atomic": "^1.0.8", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.3" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/multicast-dns": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-6.2.3.tgz", + "integrity": "sha512-ji6J5enbMyGRHIAkAOu3WdV8nggqviKCEKtXcOqfphZZtQrmHKycfynJ2V7eVPUA4NhJ6V7Wf4TmGbTwKE9B6g==", + "dependencies": { + "dns-packet": "^1.3.1", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/multicast-dns-service-types": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", + "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=" + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" + }, + "node_modules/nan": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "optional": true + }, + "node_modules/nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nanomatch/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=" + }, + "node_modules/negotiator": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", + "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + }, + "node_modules/next-tick": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", + "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" + }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" + }, + "node_modules/no-case": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.3.tgz", + "integrity": "sha512-ehY/mVQCf9BL0gKfsJBvFJen+1V//U+0HQMPrWct40ixE4jnv0bfvxDbWtAHL9EcaPEOJHVVYKoQn1TlZUB8Tw==", + "dependencies": { + "lower-case": "^2.0.1", + "tslib": "^1.10.0" + } + }, + "node_modules/node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=" + }, + "node_modules/node-libs-browser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", + "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "dependencies": { + "assert": "^1.1.1", + "browserify-zlib": "^0.2.0", + "buffer": "^4.3.0", + "console-browserify": "^1.1.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.11.0", + "domain-browser": "^1.1.1", + "events": "^3.0.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "0.0.1", + "process": "^0.11.10", + "punycode": "^1.2.4", + "querystring-es3": "^0.2.0", + "readable-stream": "^2.3.3", + "stream-browserify": "^2.0.1", + "stream-http": "^2.7.2", + "string_decoder": "^1.0.0", + "timers-browserify": "^2.0.4", + "tty-browserify": "0.0.0", + "url": "^0.11.0", + "util": "^0.11.0", + "vm-browserify": "^1.0.1" + } + }, + "node_modules/node-libs-browser/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "node_modules/node-libs-browser/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/node-libs-browser/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "node_modules/node-libs-browser/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/node-libs-browser/node_modules/util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dependencies": { + "inherits": "2.0.3" + } + }, + "node_modules/node-modules-regexp": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz", + "integrity": "sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/node-notifier": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-5.4.3.tgz", + "integrity": "sha512-M4UBGcs4jeOK9CjTsYwkvH6/MzuUmGCyTW+kCY7uO+1ZVr0+FHGdPdIf5CCLqAaxnRrWidyoQlNkMIIVwbKB8Q==", + "dependencies": { + "growly": "^1.3.0", + "is-wsl": "^1.1.0", + "semver": "^5.5.0", + "shellwords": "^0.1.1", + "which": "^1.3.0" + } + }, + "node_modules/node-notifier/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/node-releases": { + "version": "1.1.67", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.67.tgz", + "integrity": "sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg==" + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/normalize-package-data/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", + "dependencies": { + "remove-trailing-separator": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz", + "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=", + "dependencies": { + "object-assign": "^4.0.1", + "prepend-http": "^1.0.0", + "query-string": "^4.1.0", + "sort-keys": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dependencies": { + "boolbase": "~1.0.0" + } + }, + "node_modules/num2fraction": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz", + "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4=" + }, + "node_modules/nwsapi": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", + "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==" + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "engines": { + "node": "*" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=", + "dependencies": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.0.3.tgz", + "integrity": "sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/object-inspect": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.9.0.tgz", + "integrity": "sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-is": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.2.tgz", + "integrity": "sha512-5lHCz+0uufF6wZ7CRFWJN3hp8Jqblpgve06U5CMQ3f//6iDjPr2PEo9MWCjEssDsa+UZEL4PkFpr+BMop6aKzQ==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=", + "dependencies": { + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.entries": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.2.tgz", + "integrity": "sha512-BQdB9qKmb/HyNdMNWVr7O3+z5MUIx3aiegEIJqjMBbBf0YT9RRxTJSim4mzFqtyr7PDAHigq0N9dO0m0tRakQA==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5", + "has": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.fromentries": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.2.tgz", + "integrity": "sha512-r3ZiBH7MQppDJVLx6fhD618GKNG40CZYH9wgwdhKxBDDbQgjeWGGd4AtkZad84d291YxvWe7bJGuE65Anh0dxQ==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "has": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.getownpropertydescriptors": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", + "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.values": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", + "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "has": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, + "node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/open": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/open/-/open-7.2.1.tgz", + "integrity": "sha512-xbYCJib4spUdmcs0g/2mK1nKo/jO2T7INClWd/beL7PFkXRWgr8B23ssDHX/USPn2M2IjDR5UdpYs6I67SnTSA==", + "dependencies": { + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/open/node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/opn": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/opn/-/opn-5.5.0.tgz", + "integrity": "sha512-PqHpggC9bLV0VeWcdKhkpxY+3JTzetLSqTCWL/z/tFIbI6G8JCjondXklT1JinczLz2Xib62sSp0T/gKT4KksA==", + "dependencies": { + "is-wsl": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/optimize-css-assets-webpack-plugin": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/optimize-css-assets-webpack-plugin/-/optimize-css-assets-webpack-plugin-5.0.3.tgz", + "integrity": "sha512-q9fbvCRS6EYtUKKSwI87qm2IxlyJK5b4dygW1rKUBT6mMDhdG5e5bZT63v6tnJR9F9FB/H5a0HTmtw+laUBxKA==", + "dependencies": { + "cssnano": "^4.1.10", + "last-call-webpack-plugin": "^3.0.0" + } + }, + "node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/original": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/original/-/original-1.0.2.tgz", + "integrity": "sha512-hyBVl6iqqUOJ8FqRe+l/gS8H+kKYjrEndd5Pm1MfBtsEKA038HkkdbAl/72EAXGyonD/PFsvmVG+EvcIpliMBg==", + "dependencies": { + "url-parse": "^1.4.3" + } + }, + "node_modules/os-browserify": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-each-series": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-1.0.0.tgz", + "integrity": "sha1-kw89Et0fUOdDRFeiLNbwSsatf3E=", + "dependencies": { + "p-reduce": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-reduce": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-reduce/-/p-reduce-1.0.0.tgz", + "integrity": "sha1-GMKw3ZNqRpClKfgjH1ig/bakffo=", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-retry": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-3.0.1.tgz", + "integrity": "sha512-XE6G4+YTTkT2a0UWb2kjZe8xNwf8bIbnqpc/IS/idOBVhyves0mK5OJgeocjx7q5pvX/6m23xuzVPYT1uGM73w==", + "dependencies": { + "retry": "^0.12.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/parallel-transform": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", + "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==", + "dependencies": { + "cyclist": "^1.0.1", + "inherits": "^2.0.3", + "readable-stream": "^2.1.5" + } + }, + "node_modules/parallel-transform/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/parallel-transform/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/param-case": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.3.tgz", + "integrity": "sha512-VWBVyimc1+QrzappRs7waeN2YmoZFCGXWASRYX1/rGHtXqEcrGEIDm+jqIwFa2fRXNgQEwrxaYuIrX0WcAguTA==", + "dependencies": { + "dot-case": "^3.0.3", + "tslib": "^1.10.0" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parent-module/node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "dependencies": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/parse5": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz", + "integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==" + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/pascal-case": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.1.tgz", + "integrity": "sha512-XIeHKqIrsquVTQL2crjq3NfJUxmdLasn3TYOU0VBM+UX2a6ztAWBlJQBePLGY7VHW8+2dRadeIPK5+KImwTxQA==", + "dependencies": { + "no-case": "^3.0.3", + "tslib": "^1.10.0" + } + }, + "node_modules/pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-browserify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz", + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==" + }, + "node_modules/path-dirname": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", + "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=" + }, + "node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "engines": { + "node": ">=4" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" + }, + "node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "engines": { + "node": ">=4" + } + }, + "node_modules/path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + }, + "node_modules/path-to-regexp": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", + "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", + "dependencies": { + "isarray": "0.0.1" + } + }, + "node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pbkdf2": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz", + "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==", + "dependencies": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + }, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "node_modules/picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "engines": { + "node": ">=8.6" + } + }, + "node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "engines": { + "node": ">=4" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pirates": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.1.tgz", + "integrity": "sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA==", + "dependencies": { + "node-modules-regexp": "^1.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-up": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", + "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, + "node_modules/pnp-webpack-plugin": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/pnp-webpack-plugin/-/pnp-webpack-plugin-1.6.4.tgz", + "integrity": "sha512-7Wjy+9E3WwLOEL30D+m8TSTF7qJJUJLONBnwQp0518siuMxUQUbgZwssaFX+QKlZkjHZcw/IpZCt/H0srrntSg==", + "dependencies": { + "ts-pnp": "^1.1.6" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/popper.js": { + "version": "1.16.1-lts", + "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1-lts.tgz", + "integrity": "sha512-Kjw8nKRl1m+VrSFCoVGPph93W/qrSO7ZkqPpTf7F4bk/sqcfWK019dWBUpE/fBOsOQY1dks/Bmcbfn1heM/IsA==" + }, + "node_modules/portfinder": { + "version": "1.0.28", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.28.tgz", + "integrity": "sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==", + "dependencies": { + "async": "^2.6.2", + "debug": "^3.1.1", + "mkdirp": "^0.5.5" + }, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/portfinder/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss": { + "version": "7.0.35", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.35.tgz", + "integrity": "sha512-3QT8bBJeX/S5zKTTjTCIjRF3If4avAT6kqxcASlTWEtAFCb9NH0OUxNDfgZSWdP5fJnBYCMEWkIFfWeugjzYMg==", + "dependencies": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + } + }, + "node_modules/postcss-attribute-case-insensitive": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-attribute-case-insensitive/-/postcss-attribute-case-insensitive-4.0.2.tgz", + "integrity": "sha512-clkFxk/9pcdb4Vkn0hAHq3YnxBQ2p0CGD1dy24jN+reBck+EWxMbxSUqN4Yj7t0w8csl87K6p0gxBe1utkJsYA==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-selector-parser": "^6.0.2" + } + }, + "node_modules/postcss-browser-comments": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-browser-comments/-/postcss-browser-comments-3.0.0.tgz", + "integrity": "sha512-qfVjLfq7HFd2e0HW4s1dvU8X080OZdG46fFbIBFjW7US7YPDcWfRvdElvwMJr2LI6hMmD+7LnH2HcmXTs+uOig==", + "dependencies": { + "postcss": "^7" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/postcss-calc": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-7.0.3.tgz", + "integrity": "sha512-IB/EAEmZhIMEIhG7Ov4x+l47UaXOS1n2f4FBUk/aKllQhtSCxWhTzn0nJgkqN7fo/jcWySvWTSB6Syk9L+31bA==", + "dependencies": { + "postcss": "^7.0.27", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.0.2" + } + }, + "node_modules/postcss-color-functional-notation": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postcss-color-functional-notation/-/postcss-color-functional-notation-2.0.1.tgz", + "integrity": "sha512-ZBARCypjEDofW4P6IdPVTLhDNXPRn8T2s1zHbZidW6rPaaZvcnCS2soYFIQJrMZSxiePJ2XIYTlcb2ztr/eT2g==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-color-gray": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-color-gray/-/postcss-color-gray-5.0.0.tgz", + "integrity": "sha512-q6BuRnAGKM/ZRpfDascZlIZPjvwsRye7UDNalqVz3s7GDxMtqPY6+Q871liNxsonUw8oC61OG+PSaysYpl1bnw==", + "dependencies": { + "@csstools/convert-colors": "^1.4.0", + "postcss": "^7.0.5", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-color-hex-alpha": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/postcss-color-hex-alpha/-/postcss-color-hex-alpha-5.0.3.tgz", + "integrity": "sha512-PF4GDel8q3kkreVXKLAGNpHKilXsZ6xuu+mOQMHWHLPNyjiUBOr75sp5ZKJfmv1MCus5/DWUGcK9hm6qHEnXYw==", + "dependencies": { + "postcss": "^7.0.14", + "postcss-values-parser": "^2.0.1" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-color-mod-function": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/postcss-color-mod-function/-/postcss-color-mod-function-3.0.3.tgz", + "integrity": "sha512-YP4VG+xufxaVtzV6ZmhEtc+/aTXH3d0JLpnYfxqTvwZPbJhWqp8bSY3nfNzNRFLgB4XSaBA82OE4VjOOKpCdVQ==", + "dependencies": { + "@csstools/convert-colors": "^1.4.0", + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-color-rebeccapurple": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-color-rebeccapurple/-/postcss-color-rebeccapurple-4.0.1.tgz", + "integrity": "sha512-aAe3OhkS6qJXBbqzvZth2Au4V3KieR5sRQ4ptb2b2O8wgvB3SJBsdG+jsn2BZbbwekDG8nTfcCNKcSfe/lEy8g==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-colormin": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-4.0.3.tgz", + "integrity": "sha512-WyQFAdDZpExQh32j0U0feWisZ0dmOtPl44qYmJKkq9xFWY3p+4qnRzCHeNrkeRhwPHz9bQ3mo0/yVkaply0MNw==", + "dependencies": { + "browserslist": "^4.0.0", + "color": "^3.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-colormin/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-convert-values": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-4.0.1.tgz", + "integrity": "sha512-Kisdo1y77KUC0Jmn0OXU/COOJbzM8cImvw1ZFsBgBgMgb1iL23Zs/LXRe3r+EZqM3vGYKdQ2YJVQ5VkJI+zEJQ==", + "dependencies": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-convert-values/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-custom-media": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/postcss-custom-media/-/postcss-custom-media-7.0.8.tgz", + "integrity": "sha512-c9s5iX0Ge15o00HKbuRuTqNndsJUbaXdiNsksnVH8H4gdc+zbLzr/UasOwNG6CTDpLFekVY4672eWdiiWu2GUg==", + "dependencies": { + "postcss": "^7.0.14" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-custom-properties": { + "version": "8.0.11", + "resolved": "https://registry.npmjs.org/postcss-custom-properties/-/postcss-custom-properties-8.0.11.tgz", + "integrity": "sha512-nm+o0eLdYqdnJ5abAJeXp4CEU1c1k+eB2yMCvhgzsds/e0umabFrN6HoTy/8Q4K5ilxERdl/JD1LO5ANoYBeMA==", + "dependencies": { + "postcss": "^7.0.17", + "postcss-values-parser": "^2.0.1" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-custom-selectors": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/postcss-custom-selectors/-/postcss-custom-selectors-5.1.2.tgz", + "integrity": "sha512-DSGDhqinCqXqlS4R7KGxL1OSycd1lydugJ1ky4iRXPHdBRiozyMHrdu0H3o7qNOCiZwySZTUI5MV0T8QhCLu+w==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-selector-parser": "^5.0.0-rc.3" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-custom-selectors/node_modules/cssesc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-2.0.0.tgz", + "integrity": "sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-custom-selectors/node_modules/postcss-selector-parser": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz", + "integrity": "sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ==", + "dependencies": { + "cssesc": "^2.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-dir-pseudo-class": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-dir-pseudo-class/-/postcss-dir-pseudo-class-5.0.0.tgz", + "integrity": "sha512-3pm4oq8HYWMZePJY+5ANriPs3P07q+LW6FAdTlkFH2XqDdP4HeeJYMOzn0HYLhRSjBO3fhiqSwwU9xEULSrPgw==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-selector-parser": "^5.0.0-rc.3" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/postcss-dir-pseudo-class/node_modules/cssesc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-2.0.0.tgz", + "integrity": "sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-dir-pseudo-class/node_modules/postcss-selector-parser": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz", + "integrity": "sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ==", + "dependencies": { + "cssesc": "^2.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-discard-comments": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-4.0.2.tgz", + "integrity": "sha512-RJutN259iuRf3IW7GZyLM5Sw4GLTOH8FmsXBnv8Ab/Tc2k4SR4qbV4DNbyyY4+Sjo362SyDmW2DQ7lBSChrpkg==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-discard-duplicates": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-4.0.2.tgz", + "integrity": "sha512-ZNQfR1gPNAiXZhgENFfEglF93pciw0WxMkJeVmw8eF+JZBbMD7jp6C67GqJAXVZP2BWbOztKfbsdmMp/k8c6oQ==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-discard-empty": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-4.0.1.tgz", + "integrity": "sha512-B9miTzbznhDjTfjvipfHoqbWKwd0Mj+/fL5s1QOz06wufguil+Xheo4XpOnc4NqKYBCNqqEzgPv2aPBIJLox0w==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-discard-overridden": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-4.0.1.tgz", + "integrity": "sha512-IYY2bEDD7g1XM1IDEsUT4//iEYCxAmP5oDSFMVU/JVvT7gh+l4fmjciLqGgwjdWpQIdb0Che2VX00QObS5+cTg==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-double-position-gradients": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postcss-double-position-gradients/-/postcss-double-position-gradients-1.0.0.tgz", + "integrity": "sha512-G+nV8EnQq25fOI8CH/B6krEohGWnF5+3A6H/+JEpOncu5dCnkS1QQ6+ct3Jkaepw1NGVqqOZH6lqrm244mCftA==", + "dependencies": { + "postcss": "^7.0.5", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-env-function": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/postcss-env-function/-/postcss-env-function-2.0.2.tgz", + "integrity": "sha512-rwac4BuZlITeUbiBq60h/xbLzXY43qOsIErngWa4l7Mt+RaSkT7QBjXVGTcBHupykkblHMDrBFh30zchYPaOUw==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-flexbugs-fixes": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-flexbugs-fixes/-/postcss-flexbugs-fixes-4.1.0.tgz", + "integrity": "sha512-jr1LHxQvStNNAHlgco6PzY308zvLklh7SJVYuWUwyUQncofaAlD2l+P/gxKHOdqWKe7xJSkVLFF/2Tp+JqMSZA==", + "dependencies": { + "postcss": "^7.0.0" + } + }, + "node_modules/postcss-focus-visible": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-focus-visible/-/postcss-focus-visible-4.0.0.tgz", + "integrity": "sha512-Z5CkWBw0+idJHSV6+Bgf2peDOFf/x4o+vX/pwcNYrWpXFrSfTkQ3JQ1ojrq9yS+upnAlNRHeg8uEwFTgorjI8g==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-focus-within": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-focus-within/-/postcss-focus-within-3.0.0.tgz", + "integrity": "sha512-W0APui8jQeBKbCGZudW37EeMCjDeVxKgiYfIIEo8Bdh5SpB9sxds/Iq8SEuzS0Q4YFOlG7EPFulbbxujpkrV2w==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-font-variant": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-font-variant/-/postcss-font-variant-4.0.0.tgz", + "integrity": "sha512-M8BFYKOvCrI2aITzDad7kWuXXTm0YhGdP9Q8HanmN4EF1Hmcgs1KK5rSHylt/lUJe8yLxiSwWAHdScoEiIxztg==", + "dependencies": { + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-gap-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postcss-gap-properties/-/postcss-gap-properties-2.0.0.tgz", + "integrity": "sha512-QZSqDaMgXCHuHTEzMsS2KfVDOq7ZFiknSpkrPJY6jmxbugUPTuSzs/vuE5I3zv0WAS+3vhrlqhijiprnuQfzmg==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-image-set-function": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/postcss-image-set-function/-/postcss-image-set-function-3.0.1.tgz", + "integrity": "sha512-oPTcFFip5LZy8Y/whto91L9xdRHCWEMs3e1MdJxhgt4jy2WYXfhkng59fH5qLXSCPN8k4n94p1Czrfe5IOkKUw==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-initial": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postcss-initial/-/postcss-initial-3.0.2.tgz", + "integrity": "sha512-ugA2wKonC0xeNHgirR4D3VWHs2JcU08WAi1KFLVcnb7IN89phID6Qtg2RIctWbnvp1TM2BOmDtX8GGLCKdR8YA==", + "dependencies": { + "lodash.template": "^4.5.0", + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-lab-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postcss-lab-function/-/postcss-lab-function-2.0.1.tgz", + "integrity": "sha512-whLy1IeZKY+3fYdqQFuDBf8Auw+qFuVnChWjmxm/UhHWqNHZx+B99EwxTvGYmUBqe3Fjxs4L1BoZTJmPu6usVg==", + "dependencies": { + "@csstools/convert-colors": "^1.4.0", + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-load-config": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-2.1.0.tgz", + "integrity": "sha512-4pV3JJVPLd5+RueiVVB+gFOAa7GWc25XQcMp86Zexzke69mKf6Nx9LRcQywdz7yZI9n1udOxmLuAwTBypypF8Q==", + "dependencies": { + "cosmiconfig": "^5.0.0", + "import-cwd": "^2.0.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/postcss-loader": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-loader/-/postcss-loader-3.0.0.tgz", + "integrity": "sha512-cLWoDEY5OwHcAjDnkyRQzAXfs2jrKjXpO/HQFcc5b5u/r7aa471wdmChmwfnv7x2u840iat/wi0lQ5nbRgSkUA==", + "dependencies": { + "loader-utils": "^1.1.0", + "postcss": "^7.0.0", + "postcss-load-config": "^2.0.0", + "schema-utils": "^1.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-loader/node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/postcss-logical": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-logical/-/postcss-logical-3.0.0.tgz", + "integrity": "sha512-1SUKdJc2vuMOmeItqGuNaC+N8MzBWFWEkAnRnLpFYj1tGGa7NqyVBujfRtgNa2gXR+6RkGUiB2O5Vmh7E2RmiA==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-media-minmax": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-media-minmax/-/postcss-media-minmax-4.0.0.tgz", + "integrity": "sha512-fo9moya6qyxsjbFAYl97qKO9gyre3qvbMnkOZeZwlsW6XYFsvs2DMGDlchVLfAd8LHPZDxivu/+qW2SMQeTHBw==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-merge-longhand": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-4.0.11.tgz", + "integrity": "sha512-alx/zmoeXvJjp7L4mxEMjh8lxVlDFX1gqWHzaaQewwMZiVhLo42TEClKaeHbRf6J7j82ZOdTJ808RtN0ZOZwvw==", + "dependencies": { + "css-color-names": "0.0.4", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "stylehacks": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-merge-longhand/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-merge-rules": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-4.0.3.tgz", + "integrity": "sha512-U7e3r1SbvYzO0Jr3UT/zKBVgYYyhAz0aitvGIYOYK5CPmkNih+WDSsS5tvPrJ8YMQYlEMvsZIiqmn7HdFUaeEQ==", + "dependencies": { + "browserslist": "^4.0.0", + "caniuse-api": "^3.0.0", + "cssnano-util-same-parent": "^4.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0", + "vendors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-merge-rules/node_modules/postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dependencies": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss-minify-font-values": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-4.0.2.tgz", + "integrity": "sha512-j85oO6OnRU9zPf04+PZv1LYIYOprWm6IA6zkXkrJXyRveDEuQggG6tvoy8ir8ZwjLxLuGfNkCZEQG7zan+Hbtg==", + "dependencies": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-minify-font-values/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-minify-gradients": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-4.0.2.tgz", + "integrity": "sha512-qKPfwlONdcf/AndP1U8SJ/uzIJtowHlMaSioKzebAXSG4iJthlWC9iSWznQcX4f66gIWX44RSA841HTHj3wK+Q==", + "dependencies": { + "cssnano-util-get-arguments": "^4.0.0", + "is-color-stop": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-minify-gradients/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-minify-params": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-4.0.2.tgz", + "integrity": "sha512-G7eWyzEx0xL4/wiBBJxJOz48zAKV2WG3iZOqVhPet/9geefm/Px5uo1fzlHu+DOjT+m0Mmiz3jkQzVHe6wxAWg==", + "dependencies": { + "alphanum-sort": "^1.0.0", + "browserslist": "^4.0.0", + "cssnano-util-get-arguments": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "uniqs": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-minify-params/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-minify-selectors": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-4.0.2.tgz", + "integrity": "sha512-D5S1iViljXBj9kflQo4YutWnJmwm8VvIsU1GeXJGiG9j8CIg9zs4voPMdQDUmIxetUOh60VilsNzCiAFTOqu3g==", + "dependencies": { + "alphanum-sort": "^1.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-minify-selectors/node_modules/postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dependencies": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-2.0.0.tgz", + "integrity": "sha512-LaYLDNS4SG8Q5WAWqIJgdHPJrDDr/Lv775rMBFUbgjTz6j34lUznACHcdRWroPvXANP2Vj7yNK57vp9eFqzLWQ==", + "dependencies": { + "postcss": "^7.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-3.0.3.tgz", + "integrity": "sha512-e3xDq+LotiGesympRlKNgaJ0PCzoUIdpH0dj47iWAui/kyTgh3CiAr1qP54uodmJhl6p9rN6BoNcdEDVJx9RDw==", + "dependencies": { + "icss-utils": "^4.1.1", + "postcss": "^7.0.32", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-modules-scope": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-2.2.0.tgz", + "integrity": "sha512-YyEgsTMRpNd+HmyC7H/mh3y+MeFWevy7V1evVhJWewmMbjDHIbZbOXICC2y+m1xI1UVfIT1HMW/O04Hxyu9oXQ==", + "dependencies": { + "postcss": "^7.0.6", + "postcss-selector-parser": "^6.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-modules-values": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-3.0.0.tgz", + "integrity": "sha512-1//E5jCBrZ9DmRX+zCtmQtRSV6PV42Ix7Bzj9GbwJceduuf7IqP8MgeTXuRDHOWj2m0VzZD5+roFWDuU8RQjcg==", + "dependencies": { + "icss-utils": "^4.0.0", + "postcss": "^7.0.6" + } + }, + "node_modules/postcss-nesting": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/postcss-nesting/-/postcss-nesting-7.0.1.tgz", + "integrity": "sha512-FrorPb0H3nuVq0Sff7W2rnc3SmIcruVC6YwpcS+k687VxyxO33iE1amna7wHuRVzM8vfiYofXSBHNAZ3QhLvYg==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-normalize": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/postcss-normalize/-/postcss-normalize-8.0.1.tgz", + "integrity": "sha512-rt9JMS/m9FHIRroDDBGSMsyW1c0fkvOJPy62ggxSHUldJO7B195TqFMqIf+lY5ezpDcYOV4j86aUp3/XbxzCCQ==", + "dependencies": { + "@csstools/normalize.css": "^10.1.0", + "browserslist": "^4.6.2", + "postcss": "^7.0.17", + "postcss-browser-comments": "^3.0.0", + "sanitize.css": "^10.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/postcss-normalize-charset": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-4.0.1.tgz", + "integrity": "sha512-gMXCrrlWh6G27U0hF3vNvR3w8I1s2wOBILvA87iNXaPvSNo5uZAMYsZG7XjCUf1eVxuPfyL4TJ7++SGZLc9A3g==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-display-values": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-display-values/-/postcss-normalize-display-values-4.0.2.tgz", + "integrity": "sha512-3F2jcsaMW7+VtRMAqf/3m4cPFhPD3EFRgNs18u+k3lTJJlVe7d0YPO+bnwqo2xg8YiRpDXJI2u8A0wqJxMsQuQ==", + "dependencies": { + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-display-values/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-positions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-positions/-/postcss-normalize-positions-4.0.2.tgz", + "integrity": "sha512-Dlf3/9AxpxE+NF1fJxYDeggi5WwV35MXGFnnoccP/9qDtFrTArZ0D0R+iKcg5WsUd8nUYMIl8yXDCtcrT8JrdA==", + "dependencies": { + "cssnano-util-get-arguments": "^4.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-positions/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-repeat-style": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-4.0.2.tgz", + "integrity": "sha512-qvigdYYMpSuoFs3Is/f5nHdRLJN/ITA7huIoCyqqENJe9PvPmLhNLMu7QTjPdtnVf6OcYYO5SHonx4+fbJE1+Q==", + "dependencies": { + "cssnano-util-get-arguments": "^4.0.0", + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-repeat-style/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-string": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-string/-/postcss-normalize-string-4.0.2.tgz", + "integrity": "sha512-RrERod97Dnwqq49WNz8qo66ps0swYZDSb6rM57kN2J+aoyEAJfZ6bMx0sx/F9TIEX0xthPGCmeyiam/jXif0eA==", + "dependencies": { + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-string/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-timing-functions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-4.0.2.tgz", + "integrity": "sha512-acwJY95edP762e++00Ehq9L4sZCEcOPyaHwoaFOhIwWCDfik6YvqsYNxckee65JHLKzuNSSmAdxwD2Cud1Z54A==", + "dependencies": { + "cssnano-util-get-match": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-timing-functions/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-unicode": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-normalize-unicode/-/postcss-normalize-unicode-4.0.1.tgz", + "integrity": "sha512-od18Uq2wCYn+vZ/qCOeutvHjB5jm57ToxRaMeNuf0nWVHaP9Hua56QyMF6fs/4FSUnVIw0CBPsU0K4LnBPwYwg==", + "dependencies": { + "browserslist": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-unicode/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-url": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-4.0.1.tgz", + "integrity": "sha512-p5oVaF4+IHwu7VpMan/SSpmpYxcJMtkGppYf0VbdH5B6hN8YNmVyJLuY9FmLQTzY3fag5ESUUHDqM+heid0UVA==", + "dependencies": { + "is-absolute-url": "^2.0.0", + "normalize-url": "^3.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-url/node_modules/normalize-url": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-3.3.0.tgz", + "integrity": "sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/postcss-normalize-url/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-normalize-whitespace": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-normalize-whitespace/-/postcss-normalize-whitespace-4.0.2.tgz", + "integrity": "sha512-tO8QIgrsI3p95r8fyqKV+ufKlSHh9hMJqACqbv2XknufqEDhDvbguXGBBqxw9nsQoXWf0qOqppziKJKHMD4GtA==", + "dependencies": { + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-normalize-whitespace/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-ordered-values": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-4.1.2.tgz", + "integrity": "sha512-2fCObh5UanxvSxeXrtLtlwVThBvHn6MQcu4ksNT2tsaV2Fg76R2CV98W7wNSlX+5/pFwEyaDwKLLoEV7uRybAw==", + "dependencies": { + "cssnano-util-get-arguments": "^4.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-ordered-values/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-overflow-shorthand": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postcss-overflow-shorthand/-/postcss-overflow-shorthand-2.0.0.tgz", + "integrity": "sha512-aK0fHc9CBNx8jbzMYhshZcEv8LtYnBIRYQD5i7w/K/wS9c2+0NSR6B3OVMu5y0hBHYLcMGjfU+dmWYNKH0I85g==", + "dependencies": { + "postcss": "^7.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-page-break": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postcss-page-break/-/postcss-page-break-2.0.0.tgz", + "integrity": "sha512-tkpTSrLpfLfD9HvgOlJuigLuk39wVTbbd8RKcy8/ugV2bNBUW3xU+AIqyxhDrQr1VUj1RmyJrBn1YWrqUm9zAQ==", + "dependencies": { + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-place": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-place/-/postcss-place-4.0.1.tgz", + "integrity": "sha512-Zb6byCSLkgRKLODj/5mQugyuj9bvAAw9LqJJjgwz5cYryGeXfFZfSXoP1UfveccFmeq0b/2xxwcTEVScnqGxBg==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-values-parser": "^2.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-preset-env": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/postcss-preset-env/-/postcss-preset-env-6.7.0.tgz", + "integrity": "sha512-eU4/K5xzSFwUFJ8hTdTQzo2RBLbDVt83QZrAvI07TULOkmyQlnYlpwep+2yIK+K+0KlZO4BvFcleOCCcUtwchg==", + "dependencies": { + "autoprefixer": "^9.6.1", + "browserslist": "^4.6.4", + "caniuse-lite": "^1.0.30000981", + "css-blank-pseudo": "^0.1.4", + "css-has-pseudo": "^0.10.0", + "css-prefers-color-scheme": "^3.1.1", + "cssdb": "^4.4.0", + "postcss": "^7.0.17", + "postcss-attribute-case-insensitive": "^4.0.1", + "postcss-color-functional-notation": "^2.0.1", + "postcss-color-gray": "^5.0.0", + "postcss-color-hex-alpha": "^5.0.3", + "postcss-color-mod-function": "^3.0.3", + "postcss-color-rebeccapurple": "^4.0.1", + "postcss-custom-media": "^7.0.8", + "postcss-custom-properties": "^8.0.11", + "postcss-custom-selectors": "^5.1.2", + "postcss-dir-pseudo-class": "^5.0.0", + "postcss-double-position-gradients": "^1.0.0", + "postcss-env-function": "^2.0.2", + "postcss-focus-visible": "^4.0.0", + "postcss-focus-within": "^3.0.0", + "postcss-font-variant": "^4.0.0", + "postcss-gap-properties": "^2.0.0", + "postcss-image-set-function": "^3.0.1", + "postcss-initial": "^3.0.0", + "postcss-lab-function": "^2.0.1", + "postcss-logical": "^3.0.0", + "postcss-media-minmax": "^4.0.0", + "postcss-nesting": "^7.0.0", + "postcss-overflow-shorthand": "^2.0.0", + "postcss-page-break": "^2.0.0", + "postcss-place": "^4.0.1", + "postcss-pseudo-class-any-link": "^6.0.0", + "postcss-replace-overflow-wrap": "^3.0.0", + "postcss-selector-matches": "^4.0.0", + "postcss-selector-not": "^4.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-pseudo-class-any-link": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-pseudo-class-any-link/-/postcss-pseudo-class-any-link-6.0.0.tgz", + "integrity": "sha512-lgXW9sYJdLqtmw23otOzrtbDXofUdfYzNm4PIpNE322/swES3VU9XlXHeJS46zT2onFO7V1QFdD4Q9LiZj8mew==", + "dependencies": { + "postcss": "^7.0.2", + "postcss-selector-parser": "^5.0.0-rc.3" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-pseudo-class-any-link/node_modules/cssesc": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-2.0.0.tgz", + "integrity": "sha512-MsCAG1z9lPdoO/IUMLSBWBSVxVtJ1395VGIQ+Fc2gNdkQ1hNDnQdw3YhA71WJCBW1vdwA0cAnk/DnW6bqoEUYg==", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-pseudo-class-any-link/node_modules/postcss-selector-parser": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-5.0.0.tgz", + "integrity": "sha512-w+zLE5Jhg6Liz8+rQOWEAwtwkyqpfnmsinXjXg6cY7YIONZZtgvE0v2O0uhQBs0peNomOJwWRKt6JBfTdTd3OQ==", + "dependencies": { + "cssesc": "^2.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-reduce-initial": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-4.0.3.tgz", + "integrity": "sha512-gKWmR5aUulSjbzOfD9AlJiHCGH6AEVLaM0AV+aSioxUDd16qXP1PCh8d1/BGVvpdWn8k/HiK7n6TjeoXN1F7DA==", + "dependencies": { + "browserslist": "^4.0.0", + "caniuse-api": "^3.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-reduce-transforms": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-4.0.2.tgz", + "integrity": "sha512-EEVig1Q2QJ4ELpJXMZR8Vt5DQx8/mo+dGWSR7vWXqcob2gQLyQGsionYcGKATXvQzMPn6DSN1vTN7yFximdIAg==", + "dependencies": { + "cssnano-util-get-match": "^4.0.0", + "has": "^1.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-reduce-transforms/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-replace-overflow-wrap": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-replace-overflow-wrap/-/postcss-replace-overflow-wrap-3.0.0.tgz", + "integrity": "sha512-2T5hcEHArDT6X9+9dVSPQdo7QHzG4XKclFT8rU5TzJPDN7RIRTbO9c4drUISOVemLj03aezStHCR2AIcr8XLpw==", + "dependencies": { + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-safe-parser": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-safe-parser/-/postcss-safe-parser-4.0.1.tgz", + "integrity": "sha512-xZsFA3uX8MO3yAda03QrG3/Eg1LN3EPfjjf07vke/46HERLZyHrTsQ9E1r1w1W//fWEhtYNndo2hQplN2cVpCQ==", + "dependencies": { + "postcss": "^7.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/postcss-selector-matches": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-matches/-/postcss-selector-matches-4.0.0.tgz", + "integrity": "sha512-LgsHwQR/EsRYSqlwdGzeaPKVT0Ml7LAT6E75T8W8xLJY62CE4S/l03BWIt3jT8Taq22kXP08s2SfTSzaraoPww==", + "dependencies": { + "balanced-match": "^1.0.0", + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-selector-not": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-selector-not/-/postcss-selector-not-4.0.0.tgz", + "integrity": "sha512-W+bkBZRhqJaYN8XAnbbZPLWMvZD1wKTu0UxtFKdhtGjWYmxhkUneoeOhRJKdAE5V7ZTlnbHfCR+6bNwK9e1dTQ==", + "dependencies": { + "balanced-match": "^1.0.0", + "postcss": "^7.0.2" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", + "dependencies": { + "cssesc": "^3.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-svgo": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-4.0.2.tgz", + "integrity": "sha512-C6wyjo3VwFm0QgBy+Fu7gCYOkCmgmClghO+pjcxvrcBKtiKt0uCF+hvbMO1fyv5BMImRK90SMb+dwUnfbGd+jw==", + "dependencies": { + "is-svg": "^3.0.0", + "postcss": "^7.0.0", + "postcss-value-parser": "^3.0.0", + "svgo": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-svgo/node_modules/postcss-value-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", + "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" + }, + "node_modules/postcss-unique-selectors": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-4.0.1.tgz", + "integrity": "sha512-+JanVaryLo9QwZjKrmJgkI4Fn8SBgRO6WXQBJi7KiAVPlmxikB5Jzc4EvXMT2H0/m0RjrVVm9rGNhZddm/8Spg==", + "dependencies": { + "alphanum-sort": "^1.0.0", + "postcss": "^7.0.0", + "uniqs": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz", + "integrity": "sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==" + }, + "node_modules/postcss-values-parser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postcss-values-parser/-/postcss-values-parser-2.0.1.tgz", + "integrity": "sha512-2tLuBsA6P4rYTNKCXYG/71C7j1pU6pK503suYOmn4xYrQIzW+opD+7FAFNuGSdZC/3Qfy334QbeMu7MEb8gOxg==", + "dependencies": { + "flatten": "^1.0.2", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=6.14.4" + } + }, + "node_modules/postcss/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss/node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/postcss/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/postcss/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/prettier": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.2.tgz", + "integrity": "sha512-5xJQIPT8BraI7ZnaDwSbu5zLrB6vvi8hVV58yHQ+QK64qrY40dULy0HSRlQ2/2IdzeBpjhDkqdcFBnFeDEMVdg==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/pretty-bytes": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.4.1.tgz", + "integrity": "sha512-s1Iam6Gwz3JI5Hweaz4GoCD1WUNUIyzePFy5+Js2hjwGVt2Z79wNN+ZKOZ2vB6C+Xs6njyB84Z1IthQg8d9LxA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/pretty-error": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-2.1.1.tgz", + "integrity": "sha1-X0+HyPkeWuPzuoerTPXgOxoX8aM=", + "dependencies": { + "renderkid": "^2.0.1", + "utila": "~0.4" + } + }, + "node_modules/pretty-format": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-25.5.0.tgz", + "integrity": "sha512-kbo/kq2LQ/A/is0PQwsEHM7Ca6//bGPPvU6UnsdDRSKTWxT/ru/xb88v4BJf6a69H+uTytOEsTusT9ksd/1iWQ==", + "dependencies": { + "@jest/types": "^25.5.0", + "ansi-regex": "^5.0.0", + "ansi-styles": "^4.0.0", + "react-is": "^16.12.0" + }, + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/pretty-format/node_modules/@jest/types": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-25.5.0.tgz", + "integrity": "sha512-OXD0RgQ86Tu3MazKo8bnrkDRaDXXMGUqd+kTtLtK1Zb7CRzQcaSRPPPV37SvYTdevXEBVxe0HXylEjs8ibkmCw==", + "dependencies": { + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^1.1.1", + "@types/yargs": "^15.0.0", + "chalk": "^3.0.0" + }, + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/pretty-format/node_modules/@types/yargs": { + "version": "15.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.11.tgz", + "integrity": "sha512-jfcNBxHFYJ4nPIacsi3woz1+kvUO6s1CyeEhtnDHBjHUMNj5UlW2GynmnSgiJJEdNg9yW5C8lfoNRZrHGv5EqA==", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/promise": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-8.1.0.tgz", + "integrity": "sha512-W04AqnILOL/sPRXziNicCjSNRruLAuIHEOVBazepu0545DDNGYHz7ar9ZgZ1fMU8/MA4mVxp5rkBWRi6OXIy3Q==", + "dependencies": { + "asap": "~2.0.6" + } + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=" + }, + "node_modules/prompts": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.3.2.tgz", + "integrity": "sha512-Q06uKs2CkNYVID0VqwfAl9mipo99zkBv/n2JtWY89Yxa3ZabWSrs0e2KTudKVa3peLUvYXMefDqIleLPVUBZMA==", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prop-types": { + "version": "15.7.2", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz", + "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", + "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "dependencies": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + }, + "node_modules/psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" + }, + "node_modules/public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "dependencies": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/public-encrypt/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "node_modules/pumpify/node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, + "node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/query-string": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz", + "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=", + "dependencies": { + "object-assign": "^4.1.0", + "strict-uri-encode": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/querystring-es3": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", + "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/raf": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/raf/-/raf-3.4.1.tgz", + "integrity": "sha512-Sq4CW4QhwOHE8ucn6J34MqtZCeWFP2aQSmrlroYgqAV1PjStIhJXxYuTgUIfkEk7zTLjmIjLmU5q+fbD1NnOJA==", + "dependencies": { + "performance-now": "^2.1.0" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "dependencies": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", + "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==", + "dependencies": { + "bytes": "3.1.0", + "http-errors": "1.7.2", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/react": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react/-/react-16.13.1.tgz", + "integrity": "sha512-YMZQQq32xHLX0bz5Mnibv1/LHb3Sqzngu7xstSM+vrkE5Kzr9xE0yMByK5kMoTK30YVJE61WfbxIFFvfeDKT1w==", + "dependencies": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1", + "prop-types": "^15.6.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-app-polyfill": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/react-app-polyfill/-/react-app-polyfill-1.0.6.tgz", + "integrity": "sha512-OfBnObtnGgLGfweORmdZbyEz+3dgVePQBb3zipiaDsMHV1NpWm0rDFYIVXFV/AK+x4VIIfWHhrdMIeoTLyRr2g==", + "dependencies": { + "core-js": "^3.5.0", + "object-assign": "^4.1.1", + "promise": "^8.0.3", + "raf": "^3.4.1", + "regenerator-runtime": "^0.13.3", + "whatwg-fetch": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/react-dev-utils": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/react-dev-utils/-/react-dev-utils-10.2.1.tgz", + "integrity": "sha512-XxTbgJnYZmxuPtY3y/UV0D8/65NKkmaia4rXzViknVnZeVlklSh8u6TnaEYPfAi/Gh1TP4mEOXHI6jQOPbeakQ==", + "dependencies": { + "@babel/code-frame": "7.8.3", + "address": "1.1.2", + "browserslist": "4.10.0", + "chalk": "2.4.2", + "cross-spawn": "7.0.1", + "detect-port-alt": "1.1.6", + "escape-string-regexp": "2.0.0", + "filesize": "6.0.1", + "find-up": "4.1.0", + "fork-ts-checker-webpack-plugin": "3.1.1", + "global-modules": "2.0.0", + "globby": "8.0.2", + "gzip-size": "5.1.1", + "immer": "1.10.0", + "inquirer": "7.0.4", + "is-root": "2.1.0", + "loader-utils": "1.2.3", + "open": "^7.0.2", + "pkg-up": "3.1.0", + "react-error-overlay": "^6.0.7", + "recursive-readdir": "2.2.2", + "shell-quote": "1.7.2", + "strip-ansi": "6.0.0", + "text-table": "0.2.0" + }, + "engines": { + "node": ">=8.10" + } + }, + "node_modules/react-dev-utils/node_modules/@babel/code-frame": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", + "integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==", + "dependencies": { + "@babel/highlight": "^7.8.3" + } + }, + "node_modules/react-dev-utils/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/react-dev-utils/node_modules/browserslist": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.10.0.tgz", + "integrity": "sha512-TpfK0TDgv71dzuTsEAlQiHeWQ/tiPqgNZVdv046fvNtBZrjbv2O3TsWCDU0AWGJJKCF/KsjNdLzR9hXOsh/CfA==", + "dependencies": { + "caniuse-lite": "^1.0.30001035", + "electron-to-chromium": "^1.3.378", + "node-releases": "^1.1.52", + "pkg-up": "^3.1.0" + }, + "bin": { + "browserslist": "cli.js" + } + }, + "node_modules/react-dev-utils/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/react-dev-utils/node_modules/chalk/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/react-dev-utils/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/react-dev-utils/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/react-dev-utils/node_modules/cross-spawn": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", + "integrity": "sha512-u7v4o84SwFpD32Z8IIcPZ6z1/ie24O6RU3RbtL5Y316l3KuHVPx9ItBgWQ6VlfAFnRnTtMUrsQ9MUUTuEZjogg==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/react-dev-utils/node_modules/emojis-list": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz", + "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/react-dev-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/react-dev-utils/node_modules/immer": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz", + "integrity": "sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg==" + }, + "node_modules/react-dev-utils/node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/react-dev-utils/node_modules/loader-utils": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz", + "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^2.0.0", + "json5": "^1.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/react-dev-utils/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/react-dev-utils/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/react-dev-utils/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/react-dom": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.13.1.tgz", + "integrity": "sha512-81PIMmVLnCNLO/fFOQxdQkvEq/+Hfpv24XNJfpyZhTRfO0QcmQIF/PgCa1zCOj2w1hrn12MFLyaJ/G0+Mxtfag==", + "dependencies": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1", + "prop-types": "^15.6.2", + "scheduler": "^0.19.1" + } + }, + "node_modules/react-error-overlay": { + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/react-error-overlay/-/react-error-overlay-6.0.7.tgz", + "integrity": "sha512-TAv1KJFh3RhqxNvhzxj6LeT5NWklP6rDr2a0jaTfsZ5wSZWHOGeqQyejUp3xxLfPt2UpyJEcVQB/zyPcmonNFA==" + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "node_modules/react-redux": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-7.2.0.tgz", + "integrity": "sha512-EvCAZYGfOLqwV7gh849xy9/pt55rJXPwmYvI4lilPM5rUT/1NxuuN59ipdBksRVSvz0KInbPnp4IfoXJXCqiDA==", + "dependencies": { + "@babel/runtime": "^7.5.5", + "hoist-non-react-statics": "^3.3.0", + "loose-envify": "^1.4.0", + "prop-types": "^15.7.2", + "react-is": "^16.9.0" + } + }, + "node_modules/react-router": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.1.2.tgz", + "integrity": "sha512-yjEuMFy1ONK246B+rsa0cUam5OeAQ8pyclRDgpxuSCrAlJ1qN9uZ5IgyKC7gQg0w8OM50NXHEegPh/ks9YuR2A==", + "dependencies": { + "@babel/runtime": "^7.1.2", + "history": "^4.9.0", + "hoist-non-react-statics": "^3.1.0", + "loose-envify": "^1.3.1", + "mini-create-react-context": "^0.3.0", + "path-to-regexp": "^1.7.0", + "prop-types": "^15.6.2", + "react-is": "^16.6.0", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + } + }, + "node_modules/react-router-dom": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-5.1.2.tgz", + "integrity": "sha512-7BPHAaIwWpZS074UKaw1FjVdZBSVWEk8IuDXdB+OkLb8vd/WRQIpA4ag9WQk61aEfQs47wHyjWUoUGGZxpQXew==", + "dependencies": { + "@babel/runtime": "^7.1.2", + "history": "^4.9.0", + "loose-envify": "^1.3.1", + "prop-types": "^15.6.2", + "react-router": "5.1.2", + "tiny-invariant": "^1.0.2", + "tiny-warning": "^1.0.0" + } + }, + "node_modules/react-scripts": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/react-scripts/-/react-scripts-3.4.4.tgz", + "integrity": "sha512-7J7GZyF/QvZkKAZLneiOIhHozvOMHey7hO9cdO9u68jjhGZlI8hDdOm6UyuHofn6Ajc9Uji5I6Psm/nKNuWdyw==", + "dependencies": { + "@babel/core": "7.9.0", + "@svgr/webpack": "4.3.3", + "@typescript-eslint/eslint-plugin": "^2.10.0", + "@typescript-eslint/parser": "^2.10.0", + "babel-eslint": "10.1.0", + "babel-jest": "^24.9.0", + "babel-loader": "8.1.0", + "babel-plugin-named-asset-import": "^0.3.6", + "babel-preset-react-app": "^9.1.2", + "camelcase": "^5.3.1", + "case-sensitive-paths-webpack-plugin": "2.3.0", + "css-loader": "3.4.2", + "dotenv": "8.2.0", + "dotenv-expand": "5.1.0", + "eslint": "^6.6.0", + "eslint-config-react-app": "^5.2.1", + "eslint-loader": "3.0.3", + "eslint-plugin-flowtype": "4.6.0", + "eslint-plugin-import": "2.20.1", + "eslint-plugin-jsx-a11y": "6.2.3", + "eslint-plugin-react": "7.19.0", + "eslint-plugin-react-hooks": "^1.6.1", + "file-loader": "4.3.0", + "fs-extra": "^8.1.0", + "fsevents": "2.1.2", + "html-webpack-plugin": "4.0.0-beta.11", + "identity-obj-proxy": "3.0.0", + "jest": "24.9.0", + "jest-environment-jsdom-fourteen": "1.0.1", + "jest-resolve": "24.9.0", + "jest-watch-typeahead": "0.4.2", + "mini-css-extract-plugin": "0.9.0", + "optimize-css-assets-webpack-plugin": "5.0.3", + "pnp-webpack-plugin": "1.6.4", + "postcss-flexbugs-fixes": "4.1.0", + "postcss-loader": "3.0.0", + "postcss-normalize": "8.0.1", + "postcss-preset-env": "6.7.0", + "postcss-safe-parser": "4.0.1", + "react-app-polyfill": "^1.0.6", + "react-dev-utils": "^10.2.1", + "resolve": "1.15.0", + "resolve-url-loader": "3.1.2", + "sass-loader": "8.0.2", + "semver": "6.3.0", + "style-loader": "0.23.1", + "terser-webpack-plugin": "2.3.8", + "ts-pnp": "1.1.6", + "url-loader": "2.3.0", + "webpack": "4.42.0", + "webpack-dev-server": "3.11.0", + "webpack-manifest-plugin": "2.2.0", + "workbox-webpack-plugin": "4.3.1" + }, + "bin": { + "react-scripts": "bin/react-scripts.js" + }, + "engines": { + "node": ">=8.10" + }, + "optionalDependencies": { + "fsevents": "2.1.2" + }, + "peerDependencies": { + "typescript": "^3.2.1" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/react-transition-group": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", + "integrity": "sha512-Djqr7OQ2aPUiYurhPalTrVy9ddmFCCzwhqQmtN+J3+3DzLO209Fdr70QrN8Z3DsglWql6iY1lDWAfpFiBtuKGw==", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + } + }, + "node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz", + "integrity": "sha512-6etQSH7nJGsK0RbG/2TeDzZFa8shjQ1um+SwQQ5cwKy0dhSXdOncEhb1CPpvQG4h7FyOV6EB6YlV0yJvZQNAkA==", + "dependencies": { + "find-up": "^3.0.0", + "read-pkg": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/realpath-native": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-1.1.0.tgz", + "integrity": "sha512-wlgPA6cCIIg9gKz0fgAPjnzh4yR/LnXovwuo9hvyGvx3h8nX4+/iLZplfUWasXpqD8BdnGnP5njOFjkUwPzvjA==", + "dependencies": { + "util.promisify": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/recursive-readdir": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", + "integrity": "sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg==", + "dependencies": { + "minimatch": "3.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/redux": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/redux/-/redux-4.0.5.tgz", + "integrity": "sha512-VSz1uMAH24DM6MF72vcojpYPtrTUu3ByVWfPL1nPfVRb5mZVTve5GnNCUV53QM/BZ66xfWrm0CTWoM+Xlz8V1w==", + "dependencies": { + "loose-envify": "^1.4.0", + "symbol-observable": "^1.2.0" + } + }, + "node_modules/redux-thunk": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.3.0.tgz", + "integrity": "sha512-km6dclyFnmcvxhAcrQV2AkZmPQjzPDjgVlQtR0EQjxZPyJ0BnMf3in1ryuR8A2qU0HldVRfxYXbFSKlI3N7Slw==" + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==" + }, + "node_modules/regenerate-unicode-properties": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz", + "integrity": "sha512-F9DjY1vKLo/tPePDycuH3dn9H1OTPIkVD9Kz4LODu+F2C75mgjAJ7x/gwy6ZcSNRAAkhNlJSOHRe8k3p+K9WhA==", + "dependencies": { + "regenerate": "^1.4.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.7", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" + }, + "node_modules/regenerator-transform": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.14.5.tgz", + "integrity": "sha512-eOf6vka5IO151Jfsw2NO9WpGX58W6wWmefK3I1zEGr0lOD0u8rwPaNqQL1aRxUaxLeKO3ArNh3VYg1KbaD+FFw==", + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dependencies": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/regex-parser": { + "version": "2.2.11", + "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz", + "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==" + }, + "node_modules/regexp.prototype.flags": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", + "integrity": "sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/regexpu-core": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.7.1.tgz", + "integrity": "sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ==", + "dependencies": { + "regenerate": "^1.4.0", + "regenerate-unicode-properties": "^8.2.0", + "regjsgen": "^0.5.1", + "regjsparser": "^0.6.4", + "unicode-match-property-ecmascript": "^1.0.4", + "unicode-match-property-value-ecmascript": "^1.2.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regjsgen": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.5.2.tgz", + "integrity": "sha512-OFFT3MfrH90xIW8OOSyUrk6QHD5E9JOTeGodiJeBS3J6IwlgzJMNE/1bZklWz5oTg+9dCMyEetclvCVXOPoN3A==" + }, + "node_modules/regjsparser": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.6.4.tgz", + "integrity": "sha512-64O87/dPDgfk8/RQqC4gkZoGyyWFIEUTTh80CU6CWuK5vkCGyekIx+oKcEIYtP/RAxSQltCZHCNu/mdd7fqlJw==", + "dependencies": { + "jsesc": "~0.5.0" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/regjsparser/node_modules/jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", + "bin": { + "jsesc": "bin/jsesc" + } + }, + "node_modules/relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha1-VNvzd+UUQKypCkzSdGANP/LYiKk=", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/remove-trailing-separator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", + "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=" + }, + "node_modules/renderkid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-2.0.3.tgz", + "integrity": "sha512-z8CLQp7EZBPCwCnncgf9C4XAi3WR0dv+uWu/PjIyhhAb5d6IJ/QZqlHFprHeKT+59//V6BNUsLbvN8+2LarxGA==", + "dependencies": { + "css-select": "^1.1.0", + "dom-converter": "^0.2", + "htmlparser2": "^3.3.0", + "strip-ansi": "^3.0.0", + "utila": "^0.4.0" + } + }, + "node_modules/renderkid/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/renderkid/node_modules/css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "dependencies": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "node_modules/renderkid/node_modules/css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "engines": { + "node": "*" + } + }, + "node_modules/renderkid/node_modules/domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/renderkid/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/repeat-element": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", + "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/request-promise-core": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.4.tgz", + "integrity": "sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw==", + "dependencies": { + "lodash": "^4.17.19" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/request-promise-native": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.9.tgz", + "integrity": "sha512-wcW+sIUiWnKgNY0dqCpOZkUbF/I+YPi+f09JZIDa39Ec+q82CpSYniDp+ISgTTbKmnpJWASeJBPZmoxH84wt3g==", + "dependencies": { + "request-promise-core": "1.1.4", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==" + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, + "node_modules/reselect": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.0.0.tgz", + "integrity": "sha512-qUgANli03jjAyGlnbYVAV5vvnOmJnODyABz51RdBN7M4WaVu8mecZWgyQNkG8Yqe3KRGRt0l4K4B3XVEULC4CA==" + }, + "node_modules/resolve": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.0.tgz", + "integrity": "sha512-+hTmAldEGE80U2wJJDC1lebb5jWqvTYAfm3YZ1ckk1gBr0MnCqUKlwK1e+anaFljIl+F5tR5IoZcm4ZDA1zMQw==", + "dependencies": { + "path-parse": "^1.0.6" + } + }, + "node_modules/resolve-cwd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz", + "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=", + "dependencies": { + "resolve-from": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-pathname": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz", + "integrity": "sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==" + }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" + }, + "node_modules/resolve-url-loader": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-3.1.2.tgz", + "integrity": "sha512-QEb4A76c8Mi7I3xNKXlRKQSlLBwjUV/ULFMP+G7n3/7tJZ8MG5wsZ3ucxP1Jz8Vevn6fnJsxDx9cIls+utGzPQ==", + "dependencies": { + "adjust-sourcemap-loader": "3.0.0", + "camelcase": "5.3.1", + "compose-function": "3.0.3", + "convert-source-map": "1.7.0", + "es6-iterator": "2.0.3", + "loader-utils": "1.2.3", + "postcss": "7.0.21", + "rework": "1.0.1", + "rework-visit": "1.0.0", + "source-map": "0.6.1" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/resolve-url-loader/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-url-loader/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-url-loader/node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-url-loader/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/resolve-url-loader/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/resolve-url-loader/node_modules/emojis-list": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz", + "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/resolve-url-loader/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-url-loader/node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dependencies": { + "minimist": "^1.2.0" + }, + "bin": { + "json5": "lib/cli.js" + } + }, + "node_modules/resolve-url-loader/node_modules/loader-utils": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz", + "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^2.0.0", + "json5": "^1.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/resolve-url-loader/node_modules/postcss": { + "version": "7.0.21", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.21.tgz", + "integrity": "sha512-uIFtJElxJo29QC753JzhidoAhvp/e/Exezkdhfmt8AymWT6/5B7W1WmponYWkHk2eg6sONyTch0A3nkMPun3SQ==", + "dependencies": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/resolve-url-loader/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-url-loader/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "engines": { + "node": ">=0.12" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rework": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/rework/-/rework-1.0.1.tgz", + "integrity": "sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=", + "dependencies": { + "convert-source-map": "^0.3.3", + "css": "^2.0.0" + } + }, + "node_modules/rework-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rework-visit/-/rework-visit-1.0.0.tgz", + "integrity": "sha1-mUWygD8hni96ygCtuLyfZA+ELJo=" + }, + "node_modules/rework/node_modules/convert-source-map": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-0.3.5.tgz", + "integrity": "sha1-8dgClQr33SYxof6+BZZVDIarMZA=" + }, + "node_modules/rgb-regex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz", + "integrity": "sha1-wODWiC3w4jviVKR16O3UGRX+rrE=" + }, + "node_modules/rgba-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/rgba-regex/-/rgba-regex-1.0.0.tgz", + "integrity": "sha1-QzdOLiyglosO8VI0YLfXMP8i7rM=" + }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, + "node_modules/rsvp": { + "version": "4.8.5", + "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", + "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==", + "engines": { + "node": "6.* || >= 7.*" + } + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-queue": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", + "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=", + "dependencies": { + "aproba": "^1.1.1" + } + }, + "node_modules/rxjs": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.2.tgz", + "integrity": "sha512-BHdBMVoWC2sL26w//BCu3YzKT4s2jip/WhwsGEDmeKYBhKDZeYezVUnHatYB7L85v5xs0BAQmg6BEYJEKxBabg==", + "dependencies": { + "tslib": "^1.9.0" + }, + "engines": { + "npm": ">=2.0.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/sane": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/sane/-/sane-4.1.0.tgz", + "integrity": "sha512-hhbzAgTIX8O7SHfp2c8/kREfEn4qO/9q8C9beyY6+tvZ87EpoZ3i1RIEvp27YBswnNbY9mWd6paKVmKbAgLfZA==", + "dependencies": { + "@cnakazawa/watch": "^1.0.3", + "anymatch": "^2.0.0", + "capture-exit": "^2.0.0", + "exec-sh": "^0.3.2", + "execa": "^1.0.0", + "fb-watchman": "^2.0.0", + "micromatch": "^3.1.4", + "minimist": "^1.1.1", + "walker": "~1.0.5" + }, + "bin": { + "sane": "src/cli.js" + }, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/sanitize.css": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/sanitize.css/-/sanitize.css-10.0.0.tgz", + "integrity": "sha512-vTxrZz4dX5W86M6oVWVdOVe72ZiPs41Oi7Z6Km4W5Turyz28mrXSJhhEBZoRtzJWIv3833WKVwLSDWWkEfupMg==" + }, + "node_modules/sass-loader": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-8.0.2.tgz", + "integrity": "sha512-7o4dbSK8/Ol2KflEmSco4jTjQoV988bM82P9CZdmo9hR3RLnvNc0ufMNdMrB0caq38JQ/FgF4/7RcbcfKzxoFQ==", + "dependencies": { + "clone-deep": "^4.0.1", + "loader-utils": "^1.2.3", + "neo-async": "^2.6.1", + "schema-utils": "^2.6.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/sass-loader/node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/sass-loader/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sass-loader/node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, + "node_modules/saxes": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", + "integrity": "sha512-Ydydq3zC+WYDJK1+gRxRapLIED9PWeSuuS41wqyoRmzvhhh9nc+QQrVMKJYzJFULazeGhzSV0QleN2wD3boh2g==", + "dependencies": { + "xmlchars": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/scheduler": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.19.1.tgz", + "integrity": "sha512-n/zwRWRYSUj0/3g/otKDRPMh6qv2SYMWNq85IEa8iZyAv8od9zDYpGSnpBEjNgcMNq6Scbu5KfIPxNF72R/2EA==", + "dependencies": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + }, + "node_modules/schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "dependencies": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=" + }, + "node_modules/selfsigned": { + "version": "1.10.8", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz", + "integrity": "sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w==", + "dependencies": { + "node-forge": "^0.10.0" + } + }, + "node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/send": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", + "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "dependencies": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.7.2", + "mime": "1.6.0", + "ms": "2.1.1", + "on-finished": "~2.3.0", + "range-parser": "~1.2.1", + "statuses": "~1.5.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/send/node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + }, + "node_modules/serialize-javascript": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", + "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=", + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "node_modules/serve-index/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, + "node_modules/serve-static": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", + "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.17.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "node_modules/set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dependencies": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, + "node_modules/setprototypeof": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", + "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" + }, + "node_modules/sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + }, + "bin": { + "sha.js": "bin.js" + } + }, + "node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha1-WQnodLp3EG1zrEFM/sH/yofZcGA=", + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU=", + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha1-f+3fLctu23fRHvHRF6tf/fCrG2U=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shell-quote": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz", + "integrity": "sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==" + }, + "node_modules/shellwords": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz", + "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==" + }, + "node_modules/side-channel": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.3.tgz", + "integrity": "sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g==", + "dependencies": { + "es-abstract": "^1.18.0-next.0", + "object-inspect": "^1.8.0" + } + }, + "node_modules/side-channel/node_modules/es-abstract": { + "version": "1.18.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", + "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-negative-zero": "^2.0.0", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==" + }, + "node_modules/slash": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz", + "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dependencies": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/slice-ansi/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/slice-ansi/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/slice-ansi/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "engines": { + "node": ">=4" + } + }, + "node_modules/snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dependencies": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dependencies": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dependencies": { + "kind-of": "^3.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/snapdragon/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "node_modules/sockjs": { + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.20.tgz", + "integrity": "sha512-SpmVOVpdq0DJc0qArhF3E5xsxvaiqGNb73XfgBpK1y3UD5gs8DSo8aCTsuT5pX8rssdc2NDIzANwP9eCAiSdTA==", + "dependencies": { + "faye-websocket": "^0.10.0", + "uuid": "^3.4.0", + "websocket-driver": "0.6.5" + } + }, + "node_modules/sockjs-client": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.4.0.tgz", + "integrity": "sha512-5zaLyO8/nri5cua0VtOrFXBPK1jbL4+1cebT/mmKA1E1ZXOvJrII75bPu0l0k843G/+iAbhEqzyKr0w/eCCj7g==", + "dependencies": { + "debug": "^3.2.5", + "eventsource": "^1.0.7", + "faye-websocket": "~0.11.1", + "inherits": "^2.0.3", + "json3": "^3.3.2", + "url-parse": "^1.4.3" + } + }, + "node_modules/sockjs-client/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/sockjs-client/node_modules/faye-websocket": { + "version": "0.11.3", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.3.tgz", + "integrity": "sha512-D2y4bovYpzziGgbHYtGCMjlJM36vAl/y+xUyn1C+FVx8szd1E+86KwVw6XvYSzOP8iMpm1X0I4xJD+QtUb36OA==", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/sort-keys": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", + "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=", + "dependencies": { + "is-plain-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==" + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dependencies": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-url": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", + "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=" + }, + "node_modules/spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==" + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dependencies": { + "extend-shallow": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" + }, + "node_modules/sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ssri": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-7.1.0.tgz", + "integrity": "sha512-77/WrDZUWocK0mvA5NTRQyveUf+wsrIc6vyrxpS8tVvYBcX215QbafrJR3KtkpskIzoFLqqNuuYQvxaMjXJ/0g==", + "dependencies": { + "figgy-pudding": "^3.5.1", + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/stable": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", + "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==" + }, + "node_modules/stack-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-1.0.2.tgz", + "integrity": "sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=", + "dependencies": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stream-browserify": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "dependencies": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + } + }, + "node_modules/stream-browserify/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/stream-browserify/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/stream-each": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.3.tgz", + "integrity": "sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==", + "dependencies": { + "end-of-stream": "^1.1.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/stream-http": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz", + "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "dependencies": { + "builtin-status-codes": "^3.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.3.6", + "to-arraybuffer": "^1.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/stream-http/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/stream-http/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + }, + "node_modules/strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-length": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-2.0.0.tgz", + "integrity": "sha1-1A27aGo6zpYMHP/KVivyxF+DY+0=", + "dependencies": { + "astral-regex": "^1.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/string-length/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "engines": { + "node": ">=4" + } + }, + "node_modules/string-length/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string.prototype.matchall": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.2.tgz", + "integrity": "sha512-N/jp6O5fMf9os0JU3E72Qhf590RSRZU/ungsL/qJUYVTNv7hTG0P/dbPjxINVN9jpscu3nzYwKESU3P3RY5tOg==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "has-symbols": "^1.0.1", + "internal-slot": "^1.0.2", + "regexp.prototype.flags": "^1.3.0", + "side-channel": "^1.0.2" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/stringify-object": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", + "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", + "dependencies": { + "get-own-enumerable-property-symbols": "^3.0.0", + "is-obj": "^1.0.1", + "is-regexp": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/stringify-object/node_modules/is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-comments": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/strip-comments/-/strip-comments-1.0.2.tgz", + "integrity": "sha512-kL97alc47hoyIQSV165tTt9rG5dn4w1dNnBhOQ3bOU1Nc1hel09jnXANaHJ7vzHLd4Ju8kseDGzlev96pghLFw==", + "dependencies": { + "babel-extract-comments": "^1.0.0", + "babel-plugin-transform-object-rest-spread": "^6.26.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "engines": { + "node": ">=8" + } + }, + "node_modules/style-loader": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-0.23.1.tgz", + "integrity": "sha512-XK+uv9kWwhZMZ1y7mysB+zoihsEj4wneFWAS5qoiLwzW0WzSqMrrsIy+a3zkQJq0ipFtBpX5W3MqyRIBF/WFGg==", + "dependencies": { + "loader-utils": "^1.1.0", + "schema-utils": "^1.0.0" + }, + "engines": { + "node": ">= 0.12.0" + } + }, + "node_modules/style-loader/node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/stylehacks": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-4.0.3.tgz", + "integrity": "sha512-7GlLk9JwlElY4Y6a/rmbH2MhVlTyVmiJd1PfTCqFaIBEGMYNsrO/v3SeGTdhBThLg4Z+NbOk/qFMwCa+J+3p/g==", + "dependencies": { + "browserslist": "^4.0.0", + "postcss": "^7.0.0", + "postcss-selector-parser": "^3.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/stylehacks/node_modules/postcss-selector-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-3.1.2.tgz", + "integrity": "sha512-h7fJ/5uWuRVyOtkO45pnt1Ih40CEleeyCHzipqAZO2e5H20g25Y48uYnFUiShvY4rZWNJ/Bib/KVPmanaCtOhA==", + "dependencies": { + "dot-prop": "^5.2.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/svg-parser": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/svg-parser/-/svg-parser-2.0.4.tgz", + "integrity": "sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==" + }, + "node_modules/svgo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.3.2.tgz", + "integrity": "sha512-yhy/sQYxR5BkC98CY7o31VGsg014AKLEPxdfhora76l36hD9Rdy5NZA/Ocn6yayNPgSamYdtX2rFJdcv07AYVw==", + "dependencies": { + "chalk": "^2.4.1", + "coa": "^2.0.2", + "css-select": "^2.0.0", + "css-select-base-adapter": "^0.1.1", + "css-tree": "1.0.0-alpha.37", + "csso": "^4.0.2", + "js-yaml": "^3.13.1", + "mkdirp": "~0.5.1", + "object.values": "^1.1.0", + "sax": "~1.2.4", + "stable": "^0.1.8", + "unquote": "~1.1.1", + "util.promisify": "~1.0.0" + }, + "bin": { + "svgo": "bin/svgo" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/svgo/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/svgo/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/svgo/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/svgo/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/svgo/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/svgo/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" + }, + "node_modules/table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dependencies": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/table/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/table/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "engines": { + "node": ">=4" + } + }, + "node_modules/table/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tapable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", + "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/terser": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", + "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "dependencies": { + "commander": "^2.20.0", + "source-map": "~0.6.1", + "source-map-support": "~0.5.12" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-2.3.8.tgz", + "integrity": "sha512-/fKw3R+hWyHfYx7Bv6oPqmk4HGQcrWLtV3X6ggvPuwPNHSnzvVV51z6OaaCOus4YLjutYGOz3pEpbhe6Up2s1w==", + "dependencies": { + "cacache": "^13.0.1", + "find-cache-dir": "^3.3.1", + "jest-worker": "^25.4.0", + "p-limit": "^2.3.0", + "schema-utils": "^2.6.6", + "serialize-javascript": "^4.0.0", + "source-map": "^0.6.1", + "terser": "^4.6.12", + "webpack-sources": "^1.4.3" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/terser-webpack-plugin/node_modules/find-cache-dir": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.1.tgz", + "integrity": "sha512-t2GDMt3oGC/v+BMwzmllWDuJF/xcDtE5j/fCGbqDD7OLuJkj0cfh1YSA5VKPvwMeLFLNDBkwOKZ2X85jGLVftQ==", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^3.0.2", + "pkg-dir": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/jest-worker": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-25.5.0.tgz", + "integrity": "sha512-/dsSmUkIy5EBGfv/IjjqmFxrNAUpBERfGs1oHROyD7yxjG/w+t0GOJDX8O1k32ySmd7+a5IhnJU2qQFcJ4n1vw==", + "dependencies": { + "merge-stream": "^2.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">= 8.3" + } + }, + "node_modules/terser-webpack-plugin/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terser-webpack-plugin/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/terser/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/test-exclude": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", + "integrity": "sha512-M+oxtseCFO3EDtAaGH7iiej3CBkzXqFMbzqYAACdzKui4eZA+pq3tZEwChvOdNfa7xxy8BfbmgJSIr43cC/+2g==", + "dependencies": { + "glob": "^7.1.3", + "minimatch": "^3.0.4", + "read-pkg-up": "^4.0.0", + "require-main-filename": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=" + }, + "node_modules/throat": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/throat/-/throat-4.1.0.tgz", + "integrity": "sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=" + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/through2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==" + }, + "node_modules/timers-browserify": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz", + "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==", + "dependencies": { + "setimmediate": "^1.0.4" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/timsort": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/timsort/-/timsort-0.3.0.tgz", + "integrity": "sha1-QFQRqOfmM5/mTbmiNN4R3DHgK9Q=" + }, + "node_modules/tiny-invariant": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.1.0.tgz", + "integrity": "sha512-ytxQvrb1cPc9WBEI/HSeYYoGD0kWnGEOR8RY6KomWLBVhqz0RgTwVO9dLrGz7dC+nN9llyI7OKAgRq8Vq4ZBSw==" + }, + "node_modules/tiny-warning": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", + "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==" + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/tmpl": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", + "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=" + }, + "node_modules/to-arraybuffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", + "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=" + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "engines": { + "node": ">=4" + } + }, + "node_modules/to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dependencies": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", + "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/ts-pnp": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/ts-pnp/-/ts-pnp-1.1.6.tgz", + "integrity": "sha512-CrG5GqAAzMT7144Cl+UIFP7mz/iIhiy+xQ6GGcnjTezhALT02uPMRw7tgDSESgB5MsfKt55+GPWw4ir1kVtMIQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/tslib": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", + "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==" + }, + "node_modules/tsutils": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.17.1.tgz", + "integrity": "sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==", + "dependencies": { + "tslib": "^1.8.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/tty-browserify": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=" + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "node_modules/type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, + "node_modules/typeface-roboto": { + "version": "0.0.75", + "resolved": "https://registry.npmjs.org/typeface-roboto/-/typeface-roboto-0.0.75.tgz", + "integrity": "sha512-VrR/IiH00Z1tFP4vDGfwZ1esNqTiDMchBEXYY9kilT6wRGgFoCAlgkEUMHb1E3mB0FsfZhv756IF0+R+SFPfdg==" + }, + "node_modules/typescript": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", + "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz", + "integrity": "sha512-jDrNnXWHd4oHiTZnx/ZG7gtUTVp+gCcTTKr8L0HjlwphROEW3+Him+IpvC+xcJEFegapiMZyZe02CyuOnRmbnQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-1.0.4.tgz", + "integrity": "sha512-L4Qoh15vTfntsn4P1zqnHulG0LdXgjSO035fEpdtp6YxXhMT51Q6vgM5lYdG/5X3MjS+k/Y9Xw4SFCY9IkR0rg==", + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^1.0.4", + "unicode-property-aliases-ecmascript": "^1.0.4" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.2.0.tgz", + "integrity": "sha512-wjuQHGQVofmSJv1uVISKLE5zO2rNGzM/KCYZch/QQvez7C1hUhBIuZ701fYXExuufJFMPhv2SyL8CyoIfMLbIQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.1.0.tgz", + "integrity": "sha512-PqSoPh/pWetQ2phoj5RLiaqIk4kCNwoV3CI+LfGmWLKI3rE3kl1h59XpX2BjgDrmbxD9ARtQobPGU1SguCYuQg==", + "engines": { + "node": ">=4" + } + }, + "node_modules/union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dependencies": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/uniq": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", + "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=" + }, + "node_modules/uniqs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz", + "integrity": "sha1-/+3ks2slKQaW5uFl1KWe25mOawI=" + }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/unquote": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unquote/-/unquote-1.1.1.tgz", + "integrity": "sha1-j97XMk7G6IoP+LkF58CYzcCG1UQ=" + }, + "node_modules/unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=", + "dependencies": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", + "dependencies": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value/node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", + "dependencies": { + "isarray": "1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/upath": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/uri-js": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz", + "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" + }, + "node_modules/url": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url-loader": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-2.3.0.tgz", + "integrity": "sha512-goSdg8VY+7nPZKUEChZSEtW5gjbS66USIGCeSJ1OVOJ7Yfuh/36YxCwMi5HVEJh6mqUYOoy3NJ0vlOMrWsSHog==", + "dependencies": { + "loader-utils": "^1.2.3", + "mime": "^2.4.4", + "schema-utils": "^2.5.0" + }, + "engines": { + "node": ">= 8.9.0" + } + }, + "node_modules/url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/use-debounce": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/use-debounce/-/use-debounce-3.4.3.tgz", + "integrity": "sha512-nxy+opOxDccWfhMl36J5BSCTpvcj89iaQk2OZWLAtBJQj7ISCtx1gh+rFbdjGfMl6vtCZf6gke/kYvrkVfHMoA==" + }, + "node_modules/util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", + "dependencies": { + "inherits": "2.0.1" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/util.promisify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", + "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", + "dependencies": { + "define-properties": "^1.1.2", + "object.getownpropertydescriptors": "^2.0.3" + } + }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", + "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=" + }, + "node_modules/utila": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz", + "integrity": "sha1-ihagXURWV6Oupe7MWxKk+lN5dyw=" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/v8-compile-cache": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", + "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==" + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/value-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/value-equal/-/value-equal-1.0.1.tgz", + "integrity": "sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw==" + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vendors": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz", + "integrity": "sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w==" + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "engines": [ + "node >=0.6.0" + ], + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + }, + "node_modules/w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "dependencies": { + "browser-process-hrtime": "^1.0.0" + } + }, + "node_modules/w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "dependencies": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, + "node_modules/walker": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.7.tgz", + "integrity": "sha1-L3+bj9ENZ3JisYqITijRlhjgKPs=", + "dependencies": { + "makeerror": "1.0.x" + } + }, + "node_modules/watchpack": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.4.tgz", + "integrity": "sha512-aWAgTW4MoSJzZPAicljkO1hsi1oKj/RRq/OJQh2PKI2UKL04c2Bs+MBOB+BBABHTXJpf9mCwHN7ANCvYsvY2sg==", + "dependencies": { + "chokidar": "^3.4.1", + "graceful-fs": "^4.1.2", + "neo-async": "^2.5.0", + "watchpack-chokidar2": "^2.0.0" + }, + "optionalDependencies": { + "watchpack-chokidar2": "^2.0.0" + } + }, + "node_modules/watchpack-chokidar2": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz", + "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==", + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + }, + "engines": { + "node": "<8.10.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "optional": true, + "dependencies": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + } + }, + "node_modules/watchpack-chokidar2/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "optional": true, + "dependencies": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/glob-parent/node_modules/is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "optional": true, + "dependencies": { + "is-extglob": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", + "optional": true, + "dependencies": { + "binary-extensions": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "optional": true + }, + "node_modules/watchpack-chokidar2/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/watchpack-chokidar2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/watchpack-chokidar2/node_modules/readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "optional": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "node_modules/webpack": { + "version": "4.42.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.42.0.tgz", + "integrity": "sha512-EzJRHvwQyBiYrYqhyjW9AqM90dE4+s1/XtCfn7uWg6cS72zH+2VPFAlsnW0+W0cDi0XRjNKUMoJtpSi50+Ph6w==", + "dependencies": { + "@webassemblyjs/ast": "1.8.5", + "@webassemblyjs/helper-module-context": "1.8.5", + "@webassemblyjs/wasm-edit": "1.8.5", + "@webassemblyjs/wasm-parser": "1.8.5", + "acorn": "^6.2.1", + "ajv": "^6.10.2", + "ajv-keywords": "^3.4.1", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^4.1.0", + "eslint-scope": "^4.0.3", + "json-parse-better-errors": "^1.0.2", + "loader-runner": "^2.4.0", + "loader-utils": "^1.2.3", + "memory-fs": "^0.4.1", + "micromatch": "^3.1.10", + "mkdirp": "^0.5.1", + "neo-async": "^2.6.1", + "node-libs-browser": "^2.2.1", + "schema-utils": "^1.0.0", + "tapable": "^1.1.3", + "terser-webpack-plugin": "^1.4.3", + "watchpack": "^1.6.0", + "webpack-sources": "^1.4.1" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-3.7.2.tgz", + "integrity": "sha512-1xC42LxbYoqLNAhV6YzTYacicgMZQTqRd27Sim9wn5hJrX3I5nxYy1SxSd4+gjUFsz1dQFj+yEe6zEVmSkeJjw==", + "dependencies": { + "memory-fs": "^0.4.1", + "mime": "^2.4.4", + "mkdirp": "^0.5.1", + "range-parser": "^1.2.1", + "webpack-log": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/webpack-dev-server": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-3.11.0.tgz", + "integrity": "sha512-PUxZ+oSTxogFQgkTtFndEtJIPNmml7ExwufBZ9L2/Xyyd5PnOL5UreWe5ZT7IU25DSdykL9p1MLQzmLh2ljSeg==", + "dependencies": { + "ansi-html": "0.0.7", + "bonjour": "^3.5.0", + "chokidar": "^2.1.8", + "compression": "^1.7.4", + "connect-history-api-fallback": "^1.6.0", + "debug": "^4.1.1", + "del": "^4.1.1", + "express": "^4.17.1", + "html-entities": "^1.3.1", + "http-proxy-middleware": "0.19.1", + "import-local": "^2.0.0", + "internal-ip": "^4.3.0", + "ip": "^1.1.5", + "is-absolute-url": "^3.0.3", + "killable": "^1.0.1", + "loglevel": "^1.6.8", + "opn": "^5.5.0", + "p-retry": "^3.0.1", + "portfinder": "^1.0.26", + "schema-utils": "^1.0.0", + "selfsigned": "^1.10.7", + "semver": "^6.3.0", + "serve-index": "^1.9.1", + "sockjs": "0.3.20", + "sockjs-client": "1.4.0", + "spdy": "^4.0.2", + "strip-ansi": "^3.0.1", + "supports-color": "^6.1.0", + "url": "^0.11.0", + "webpack-dev-middleware": "^3.7.2", + "webpack-log": "^2.0.0", + "ws": "^6.2.1", + "yargs": "^13.3.2" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 6.11.5" + } + }, + "node_modules/webpack-dev-server/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dependencies": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + }, + "optionalDependencies": { + "fsevents": "^1.2.7" + } + }, + "node_modules/webpack-dev-server/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, + "node_modules/webpack-dev-server/node_modules/glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "dependencies": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + } + }, + "node_modules/webpack-dev-server/node_modules/glob-parent/node_modules/is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "dependencies": { + "is-extglob": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "engines": { + "node": ">=4" + } + }, + "node_modules/webpack-dev-server/node_modules/is-absolute-url": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-3.0.3.tgz", + "integrity": "sha512-opmNIX7uFnS96NtPmhWQgQx6/NYFgsUXYMllcfzwWKUMwfo8kku1TvE6hkNcH+Q1ts5cMVrsY7j0bxXQDciu9Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/webpack-dev-server/node_modules/is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", + "dependencies": { + "binary-extensions": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/webpack-dev-server/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/webpack-dev-server/node_modules/readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dependencies": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/webpack-dev-server/node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/webpack-dev-server/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack-dev-server/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/webpack-dev-server/node_modules/ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/webpack-log": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-2.0.0.tgz", + "integrity": "sha512-cX8G2vR/85UYG59FgkoMamwHUIkSSlV3bBMRsbxVXVUk2j6NleCKjQ/WE9eYg9WY4w25O9w8wKP4rzNZFmUcUg==", + "dependencies": { + "ansi-colors": "^3.0.0", + "uuid": "^3.3.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/webpack-manifest-plugin": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/webpack-manifest-plugin/-/webpack-manifest-plugin-2.2.0.tgz", + "integrity": "sha512-9S6YyKKKh/Oz/eryM1RyLVDVmy3NSPV0JXMRhZ18fJsq+AwGxUY34X54VNwkzYcEmEkDwNxuEOboCZEebJXBAQ==", + "dependencies": { + "fs-extra": "^7.0.0", + "lodash": ">=3.5 <5", + "object.entries": "^1.1.0", + "tapable": "^1.0.0" + }, + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/webpack-manifest-plugin/node_modules/fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/webpack-sources": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", + "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dependencies": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + } + }, + "node_modules/webpack-sources/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack/node_modules/acorn": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/webpack/node_modules/cacache": { + "version": "12.0.4", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", + "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", + "dependencies": { + "bluebird": "^3.5.5", + "chownr": "^1.1.1", + "figgy-pudding": "^3.5.1", + "glob": "^7.1.4", + "graceful-fs": "^4.1.15", + "infer-owner": "^1.0.3", + "lru-cache": "^5.1.1", + "mississippi": "^3.0.0", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "promise-inflight": "^1.0.1", + "rimraf": "^2.6.3", + "ssri": "^6.0.1", + "unique-filename": "^1.1.1", + "y18n": "^4.0.0" + } + }, + "node_modules/webpack/node_modules/eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dependencies": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/webpack/node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/webpack/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/webpack/node_modules/ssri": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", + "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "dependencies": { + "figgy-pudding": "^3.5.1" + } + }, + "node_modules/webpack/node_modules/terser-webpack-plugin": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz", + "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==", + "dependencies": { + "cacache": "^12.0.2", + "find-cache-dir": "^2.1.0", + "is-wsl": "^1.1.0", + "schema-utils": "^1.0.0", + "serialize-javascript": "^4.0.0", + "source-map": "^0.6.1", + "terser": "^4.1.2", + "webpack-sources": "^1.4.0", + "worker-farm": "^1.7.0" + }, + "engines": { + "node": ">= 6.9.0" + } + }, + "node_modules/websocket-driver": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.6.5.tgz", + "integrity": "sha1-XLJVbOuF9Dc8bYI4qmkchFThOjY=", + "dependencies": { + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "dependencies": { + "iconv-lite": "0.4.24" + } + }, + "node_modules/whatwg-fetch": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.0.tgz", + "integrity": "sha512-rsum2ulz2iuZH08mJkT0Yi6JnKhwdw4oeyMjokgxd+mmqYSd9cPpOQf01TIWgjxG/U4+QR+AwKq6lSbXVxkyoQ==" + }, + "node_modules/whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "node_modules/whatwg-url": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-6.5.0.tgz", + "integrity": "sha512-rhRZRqx/TLJQWUpQ6bmrt2UV4f0HCQ463yQuONJqC6fO2VoEb1pTYddbe59SkYq87aoM5A3bdhMZiUiVws+fzQ==", + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/workbox-background-sync": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-background-sync/-/workbox-background-sync-4.3.1.tgz", + "integrity": "sha512-1uFkvU8JXi7L7fCHVBEEnc3asPpiAL33kO495UMcD5+arew9IbKW2rV5lpzhoWcm/qhGB89YfO4PmB/0hQwPRg==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-broadcast-update": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-broadcast-update/-/workbox-broadcast-update-4.3.1.tgz", + "integrity": "sha512-MTSfgzIljpKLTBPROo4IpKjESD86pPFlZwlvVG32Kb70hW+aob4Jxpblud8EhNb1/L5m43DUM4q7C+W6eQMMbA==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-build": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-build/-/workbox-build-4.3.1.tgz", + "integrity": "sha512-UHdwrN3FrDvicM3AqJS/J07X0KXj67R8Cg0waq1MKEOqzo89ap6zh6LmaLnRAjpB+bDIz+7OlPye9iii9KBnxw==", + "dependencies": { + "@babel/runtime": "^7.3.4", + "@hapi/joi": "^15.0.0", + "common-tags": "^1.8.0", + "fs-extra": "^4.0.2", + "glob": "^7.1.3", + "lodash.template": "^4.4.0", + "pretty-bytes": "^5.1.0", + "stringify-object": "^3.3.0", + "strip-comments": "^1.0.2", + "workbox-background-sync": "^4.3.1", + "workbox-broadcast-update": "^4.3.1", + "workbox-cacheable-response": "^4.3.1", + "workbox-core": "^4.3.1", + "workbox-expiration": "^4.3.1", + "workbox-google-analytics": "^4.3.1", + "workbox-navigation-preload": "^4.3.1", + "workbox-precaching": "^4.3.1", + "workbox-range-requests": "^4.3.1", + "workbox-routing": "^4.3.1", + "workbox-strategies": "^4.3.1", + "workbox-streams": "^4.3.1", + "workbox-sw": "^4.3.1", + "workbox-window": "^4.3.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/workbox-build/node_modules/fs-extra": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz", + "integrity": "sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg==", + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "node_modules/workbox-cacheable-response": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-cacheable-response/-/workbox-cacheable-response-4.3.1.tgz", + "integrity": "sha512-Rp5qlzm6z8IOvnQNkCdO9qrDgDpoPNguovs0H8C+wswLuPgSzSp9p2afb5maUt9R1uTIwOXrVQMmPfPypv+npw==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-core": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-core/-/workbox-core-4.3.1.tgz", + "integrity": "sha512-I3C9jlLmMKPxAC1t0ExCq+QoAMd0vAAHULEgRZ7kieCdUd919n53WC0AfvokHNwqRhGn+tIIj7vcb5duCjs2Kg==" + }, + "node_modules/workbox-expiration": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-expiration/-/workbox-expiration-4.3.1.tgz", + "integrity": "sha512-vsJLhgQsQouv9m0rpbXubT5jw0jMQdjpkum0uT+d9tTwhXcEZks7qLfQ9dGSaufTD2eimxbUOJfWLbNQpIDMPw==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-google-analytics": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-google-analytics/-/workbox-google-analytics-4.3.1.tgz", + "integrity": "sha512-xzCjAoKuOb55CBSwQrbyWBKqp35yg1vw9ohIlU2wTy06ZrYfJ8rKochb1MSGlnoBfXGWss3UPzxR5QL5guIFdg==", + "dependencies": { + "workbox-background-sync": "^4.3.1", + "workbox-core": "^4.3.1", + "workbox-routing": "^4.3.1", + "workbox-strategies": "^4.3.1" + } + }, + "node_modules/workbox-navigation-preload": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-navigation-preload/-/workbox-navigation-preload-4.3.1.tgz", + "integrity": "sha512-K076n3oFHYp16/C+F8CwrRqD25GitA6Rkd6+qAmLmMv1QHPI2jfDwYqrytOfKfYq42bYtW8Pr21ejZX7GvALOw==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-precaching": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-precaching/-/workbox-precaching-4.3.1.tgz", + "integrity": "sha512-piSg/2csPoIi/vPpp48t1q5JLYjMkmg5gsXBQkh/QYapCdVwwmKlU9mHdmy52KsDGIjVaqEUMFvEzn2LRaigqQ==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-range-requests": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-range-requests/-/workbox-range-requests-4.3.1.tgz", + "integrity": "sha512-S+HhL9+iTFypJZ/yQSl/x2Bf5pWnbXdd3j57xnb0V60FW1LVn9LRZkPtneODklzYuFZv7qK6riZ5BNyc0R0jZA==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-routing": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-routing/-/workbox-routing-4.3.1.tgz", + "integrity": "sha512-FkbtrODA4Imsi0p7TW9u9MXuQ5P4pVs1sWHK4dJMMChVROsbEltuE79fBoIk/BCztvOJ7yUpErMKa4z3uQLX+g==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-strategies": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-strategies/-/workbox-strategies-4.3.1.tgz", + "integrity": "sha512-F/+E57BmVG8dX6dCCopBlkDvvhg/zj6VDs0PigYwSN23L8hseSRwljrceU2WzTvk/+BSYICsWmRq5qHS2UYzhw==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-streams": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-streams/-/workbox-streams-4.3.1.tgz", + "integrity": "sha512-4Kisis1f/y0ihf4l3u/+ndMkJkIT4/6UOacU3A4BwZSAC9pQ9vSvJpIi/WFGQRH/uPXvuVjF5c2RfIPQFSS2uA==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/workbox-sw": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-sw/-/workbox-sw-4.3.1.tgz", + "integrity": "sha512-0jXdusCL2uC5gM3yYFT6QMBzKfBr2XTk0g5TPAV4y8IZDyVNDyj1a8uSXy3/XrvkVTmQvLN4O5k3JawGReXr9w==" + }, + "node_modules/workbox-webpack-plugin": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-webpack-plugin/-/workbox-webpack-plugin-4.3.1.tgz", + "integrity": "sha512-gJ9jd8Mb8wHLbRz9ZvGN57IAmknOipD3W4XNE/Lk/4lqs5Htw4WOQgakQy/o/4CoXQlMCYldaqUg+EJ35l9MEQ==", + "dependencies": { + "@babel/runtime": "^7.0.0", + "json-stable-stringify": "^1.0.1", + "workbox-build": "^4.3.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/workbox-window": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/workbox-window/-/workbox-window-4.3.1.tgz", + "integrity": "sha512-C5gWKh6I58w3GeSc0wp2Ne+rqVw8qwcmZnQGpjiek8A2wpbxSJb1FdCoQVO+jDJs35bFgo/WETgl1fqgsxN0Hg==", + "dependencies": { + "workbox-core": "^4.3.1" + } + }, + "node_modules/worker-farm": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", + "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", + "dependencies": { + "errno": "~0.1.7" + } + }, + "node_modules/worker-rpc": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/worker-rpc/-/worker-rpc-0.1.1.tgz", + "integrity": "sha512-P1WjMrUB3qgJNI9jfmpZ/htmBEjFh//6l/5y8SD9hg1Ef5zTTVVoRjTrTEzPrNBQvmhMxkoTsjOXN10GWU7aCg==", + "dependencies": { + "microevent.ts": "~0.1.1" + } + }, + "node_modules/wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dependencies": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "engines": { + "node": ">=4" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "node_modules/write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dependencies": { + "mkdirp": "^0.5.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/write-file-atomic": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.1.tgz", + "integrity": "sha512-TGHFeZEZMnv+gBFRfjAcxL5bPHrsGKtnb4qsFAws7/vlh+QfwAaySIw4AXP9ZskTTh5GWu3FLuJhsWVdiJPGvg==", + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "node_modules/ws": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", + "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, + "node_modules/xregexp": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.3.0.tgz", + "integrity": "sha512-7jXDIFXh5yJ/orPn4SXjuVrWWoi4Cr8jfV1eHv9CixKSbU+jY4mxfrBwAuDvupPNKpMUY+FeIqsVw/JLT9+B8g==", + "dependencies": { + "@babel/runtime-corejs3": "^7.8.3" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==" + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dependencies": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + } + }, + "node_modules/yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "engines": { + "node": ">=4" + } + }, + "node_modules/yargs/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + } + }, "dependencies": { "@babel/code-frame": { "version": "7.10.4", @@ -13,21 +18182,9 @@ } }, "@babel/compat-data": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.11.0.tgz", - "integrity": "sha512-TPSvJfv73ng0pfnEOh17bYMPQbI95+nGWc71Ss4vZdRBHTDqmM9Z8ZV4rYz8Ks7sfzc95n30k6ODIq5UGnXcYQ==", - "requires": { - "browserslist": "^4.12.0", - "invariant": "^2.2.4", - "semver": "^5.5.0" - }, - "dependencies": { - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" - } - } + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.12.7.tgz", + "integrity": "sha512-YaxPMGs/XIWtYqrdEOZOCPsVWfEoriXopnsz3/i7apYPXQ3698UFhS6dVT1KN5qOsWmVgw/FOrmQgpRaZayGsw==" }, "@babel/core": { "version": "7.9.0", @@ -60,11 +18217,11 @@ } }, "@babel/generator": { - "version": "7.11.4", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.11.4.tgz", - "integrity": "sha512-Rn26vueFx0eOoz7iifCN2UHT6rGtnkSGWSoDRIy8jZN3B91PzeSULbswfLoOWuTuAcNwpG/mxy+uCTDnZ9Mp1g==", + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.12.5.tgz", + "integrity": "sha512-m16TQQJ8hPt7E+OS/XVQg/7U184MLXtvuGbCdA7na61vha+ImkyyNM/9DDA0unYCVZn3ZOhng+qz48/KBOT96A==", "requires": { - "@babel/types": "^7.11.0", + "@babel/types": "^7.12.5", "jsesc": "^2.5.1", "source-map": "^0.5.0" } @@ -96,24 +18253,23 @@ } }, "@babel/helper-builder-react-jsx-experimental": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/helper-builder-react-jsx-experimental/-/helper-builder-react-jsx-experimental-7.10.5.tgz", - "integrity": "sha512-Buewnx6M4ttG+NLkKyt7baQn7ScC/Td+e99G914fRU8fGIUivDDgVIQeDHFa5e4CRSJQt58WpNHhsAZgtzVhsg==", + "version": "7.12.4", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-react-jsx-experimental/-/helper-builder-react-jsx-experimental-7.12.4.tgz", + "integrity": "sha512-AjEa0jrQqNk7eDQOo0pTfUOwQBMF+xVqrausQwT9/rTKy0g04ggFNaJpaE09IQMn9yExluigWMJcj0WC7bq+Og==", "requires": { "@babel/helper-annotate-as-pure": "^7.10.4", - "@babel/helper-module-imports": "^7.10.4", - "@babel/types": "^7.10.5" + "@babel/helper-module-imports": "^7.12.1", + "@babel/types": "^7.12.1" } }, "@babel/helper-compilation-targets": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.10.4.tgz", - "integrity": "sha512-a3rYhlsGV0UHNDvrtOXBg8/OpfV0OKTkxKPzIplS1zpx7CygDcWWxckxZeDd3gzPzC4kUT0A4nVFDK0wGMh4MQ==", + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.12.5.tgz", + "integrity": "sha512-+qH6NrscMolUlzOYngSBMIOQpKUGPPsc61Bu5W10mg84LxZ7cmvnBHzARKbDoFxVvqqAbj6Tg6N7bSrWSPXMyw==", "requires": { - "@babel/compat-data": "^7.10.4", - "browserslist": "^4.12.0", - "invariant": "^2.2.4", - "levenary": "^1.1.1", + "@babel/compat-data": "^7.12.5", + "@babel/helper-validator-option": "^7.12.1", + "browserslist": "^4.14.5", "semver": "^5.5.0" }, "dependencies": { @@ -125,26 +18281,24 @@ } }, "@babel/helper-create-class-features-plugin": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.10.5.tgz", - "integrity": "sha512-0nkdeijB7VlZoLT3r/mY3bUkw3T8WG/hNw+FATs/6+pG2039IJWjTYL0VTISqsNHMUTEnwbVnc89WIJX9Qed0A==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.12.1.tgz", + "integrity": "sha512-hkL++rWeta/OVOBTRJc9a5Azh5mt5WgZUGAKMD8JM141YsE08K//bp1unBBieO6rUKkIPyUE0USQ30jAy3Sk1w==", "requires": { "@babel/helper-function-name": "^7.10.4", - "@babel/helper-member-expression-to-functions": "^7.10.5", + "@babel/helper-member-expression-to-functions": "^7.12.1", "@babel/helper-optimise-call-expression": "^7.10.4", - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-replace-supers": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", "@babel/helper-split-export-declaration": "^7.10.4" } }, "@babel/helper-create-regexp-features-plugin": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.10.4.tgz", - "integrity": "sha512-2/hu58IEPKeoLF45DBwx3XFqsbCXmkdAay4spVr2x0jYgRxrSNp+ePwvSsy9g6YSaNDcKIQVPXk1Ov8S2edk2g==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz", + "integrity": "sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ==", "requires": { "@babel/helper-annotate-as-pure": "^7.10.4", - "@babel/helper-regex": "^7.10.4", - "regexpu-core": "^4.7.0" + "regexpu-core": "^4.7.1" } }, "@babel/helper-define-map": { @@ -192,32 +18346,34 @@ } }, "@babel/helper-member-expression-to-functions": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.11.0.tgz", - "integrity": "sha512-JbFlKHFntRV5qKw3YC0CvQnDZ4XMwgzzBbld7Ly4Mj4cbFy3KywcR8NtNctRToMWJOVvLINJv525Gd6wwVEx/Q==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz", + "integrity": "sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw==", "requires": { - "@babel/types": "^7.11.0" + "@babel/types": "^7.12.7" } }, "@babel/helper-module-imports": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.4.tgz", - "integrity": "sha512-nEQJHqYavI217oD9+s5MUBzk6x1IlvoS9WTPfgG43CbMEeStE0v+r+TucWdx8KFGowPGvyOkDT9+7DHedIDnVw==", + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz", + "integrity": "sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA==", "requires": { - "@babel/types": "^7.10.4" + "@babel/types": "^7.12.5" } }, "@babel/helper-module-transforms": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.11.0.tgz", - "integrity": "sha512-02EVu8COMuTRO1TAzdMtpBPbe6aQ1w/8fePD2YgQmxZU4gpNWaL9gK3Jp7dxlkUlUCJOTaSeA+Hrm1BRQwqIhg==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.12.1.tgz", + "integrity": "sha512-QQzehgFAZ2bbISiCpmVGfiGux8YVFXQ0abBic2Envhej22DVXV9nCFaS5hIQbkyo1AdGb+gNME2TSh3hYJVV/w==", "requires": { - "@babel/helper-module-imports": "^7.10.4", - "@babel/helper-replace-supers": "^7.10.4", - "@babel/helper-simple-access": "^7.10.4", + "@babel/helper-module-imports": "^7.12.1", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-simple-access": "^7.12.1", "@babel/helper-split-export-declaration": "^7.11.0", + "@babel/helper-validator-identifier": "^7.10.4", "@babel/template": "^7.10.4", - "@babel/types": "^7.11.0", + "@babel/traverse": "^7.12.1", + "@babel/types": "^7.12.1", "lodash": "^4.17.19" } }, @@ -234,51 +18390,41 @@ "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" }, - "@babel/helper-regex": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/helper-regex/-/helper-regex-7.10.5.tgz", - "integrity": "sha512-68kdUAzDrljqBrio7DYAEgCoJHxppJOERHOgOrDN7WjOzP0ZQ1LsSDRXcemzVZaLvjaJsJEESb6qt+znNuENDg==", - "requires": { - "lodash": "^4.17.19" - } - }, "@babel/helper-remap-async-to-generator": { - "version": "7.11.4", - "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.11.4.tgz", - "integrity": "sha512-tR5vJ/vBa9wFy3m5LLv2faapJLnDFxNWff2SAYkSE4rLUdbp7CdObYFgI7wK4T/Mj4UzpjPwzR8Pzmr5m7MHGA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.12.1.tgz", + "integrity": "sha512-9d0KQCRM8clMPcDwo8SevNs+/9a8yWVVmaE80FGJcEP8N1qToREmWEGnBn8BUlJhYRFz6fqxeRL1sl5Ogsed7A==", "requires": { "@babel/helper-annotate-as-pure": "^7.10.4", "@babel/helper-wrap-function": "^7.10.4", - "@babel/template": "^7.10.4", - "@babel/types": "^7.10.4" + "@babel/types": "^7.12.1" } }, "@babel/helper-replace-supers": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.10.4.tgz", - "integrity": "sha512-sPxZfFXocEymYTdVK1UNmFPBN+Hv5mJkLPsYWwGBxZAxaWfFu+xqp7b6qWD0yjNuNL2VKc6L5M18tOXUP7NU0A==", + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.12.5.tgz", + "integrity": "sha512-5YILoed0ZyIpF4gKcpZitEnXEJ9UoDRki1Ey6xz46rxOzfNMAhVIJMoune1hmPVxh40LRv1+oafz7UsWX+vyWA==", "requires": { - "@babel/helper-member-expression-to-functions": "^7.10.4", + "@babel/helper-member-expression-to-functions": "^7.12.1", "@babel/helper-optimise-call-expression": "^7.10.4", - "@babel/traverse": "^7.10.4", - "@babel/types": "^7.10.4" + "@babel/traverse": "^7.12.5", + "@babel/types": "^7.12.5" } }, "@babel/helper-simple-access": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.10.4.tgz", - "integrity": "sha512-0fMy72ej/VEvF8ULmX6yb5MtHG4uH4Dbd6I/aHDb/JVg0bbivwt9Wg+h3uMvX+QSFtwr5MeItvazbrc4jtRAXw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.12.1.tgz", + "integrity": "sha512-OxBp7pMrjVewSSC8fXDFrHrBcJATOOFssZwv16F3/6Xtc138GHybBfPbm9kfiqQHKhYQrlamWILwlDCeyMFEaA==", "requires": { - "@babel/template": "^7.10.4", - "@babel/types": "^7.10.4" + "@babel/types": "^7.12.1" } }, "@babel/helper-skip-transparent-expression-wrappers": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.11.0.tgz", - "integrity": "sha512-0XIdiQln4Elglgjbwo9wuJpL/K7AGCY26kmEt0+pRP0TAj4jjyNq1MjoRvikrTVqKcx4Gysxt4cXvVFXP/JO2Q==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.12.1.tgz", + "integrity": "sha512-Mf5AUuhG1/OCChOJ/HcADmvcHM42WJockombn8ATJG3OnyiSxBK/Mm5x78BQWvmtXZKHgbjdGL2kin/HOLlZGA==", "requires": { - "@babel/types": "^7.11.0" + "@babel/types": "^7.12.1" } }, "@babel/helper-split-export-declaration": { @@ -294,10 +18440,15 @@ "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==" }, + "@babel/helper-validator-option": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.12.1.tgz", + "integrity": "sha512-YpJabsXlJVWP0USHjnC/AQDTLlZERbON577YUVO/wLpqyj6HAtVYnWaQaN0iUN+1/tWn3c+uKKXjRut5115Y2A==" + }, "@babel/helper-wrap-function": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.10.4.tgz", - "integrity": "sha512-6py45WvEF0MhiLrdxtRjKjufwLL1/ob2qDJgg5JgNdojBAZSAKnAjkyOCNug6n+OBl4VW76XjvgSFTdaMcW0Ug==", + "version": "7.12.3", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.12.3.tgz", + "integrity": "sha512-Cvb8IuJDln3rs6tzjW3Y8UeelAOdnpB8xtQ4sme2MSZ9wOxrbThporC0y/EtE16VAtoyEfLM404Xr1e0OOp+ow==", "requires": { "@babel/helper-function-name": "^7.10.4", "@babel/template": "^7.10.4", @@ -372,29 +18523,20 @@ } }, "@babel/parser": { - "version": "7.11.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.11.4.tgz", - "integrity": "sha512-MggwidiH+E9j5Sh8pbrX5sJvMcsqS5o+7iB42M9/k0CD63MjYbdP4nhSh7uB5wnv2/RVzTZFTxzF/kIa5mrCqA==" + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.12.7.tgz", + "integrity": "sha512-oWR02Ubp4xTLCAqPRiNIuMVgNO5Aif/xpXtabhzW2HWUD47XJsAB4Zd/Rg30+XeQA3juXigV7hlquOTmwqLiwg==" }, "@babel/plugin-proposal-async-generator-functions": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.10.5.tgz", - "integrity": "sha512-cNMCVezQbrRGvXJwm9fu/1sJj9bHdGAgKodZdLqOQIpfoH3raqmRPBM17+lh7CzhiKRRBrGtZL9WcjxSoGYUSg==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.1.tgz", + "integrity": "sha512-d+/o30tJxFxrA1lhzJqiUcEJdI6jKlNregCv5bASeGf2Q4MXmnwH7viDo7nhx1/ohf09oaH8j1GVYG/e3Yqk6A==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-remap-async-to-generator": "^7.10.4", + "@babel/helper-remap-async-to-generator": "^7.12.1", "@babel/plugin-syntax-async-generators": "^7.8.0" } }, - "@babel/plugin-proposal-class-properties": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.10.4.tgz", - "integrity": "sha512-vhwkEROxzcHGNu2mzUC0OFFNXdZ4M23ib8aRRcJSsW8BZK9pQMD7QB7csl97NBbgGZO7ZyHUyKDnxzOaP4IrCg==", - "requires": { - "@babel/helper-create-class-features-plugin": "^7.10.4", - "@babel/helper-plugin-utils": "^7.10.4" - } - }, "@babel/plugin-proposal-decorators": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.8.3.tgz", @@ -406,103 +18548,76 @@ } }, "@babel/plugin-proposal-dynamic-import": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.10.4.tgz", - "integrity": "sha512-up6oID1LeidOOASNXgv/CFbgBqTuKJ0cJjz6An5tWD+NVBNlp3VNSBxv2ZdU7SYl3NxJC7agAQDApZusV6uFwQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.12.1.tgz", + "integrity": "sha512-a4rhUSZFuq5W8/OO8H7BL5zspjnc1FLd9hlOxIK/f7qG4a0qsqk8uvF/ywgBA8/OmjsapjpvaEOYItfGG1qIvQ==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-dynamic-import": "^7.8.0" } }, - "@babel/plugin-proposal-export-namespace-from": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.10.4.tgz", - "integrity": "sha512-aNdf0LY6/3WXkhh0Fdb6Zk9j1NMD8ovj3F6r0+3j837Pn1S1PdNtcwJ5EG9WkVPNHPxyJDaxMaAOVq4eki0qbg==", - "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-export-namespace-from": "^7.8.3" - } - }, "@babel/plugin-proposal-json-strings": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.10.4.tgz", - "integrity": "sha512-fCL7QF0Jo83uy1K0P2YXrfX11tj3lkpN7l4dMv9Y9VkowkhkQDwFHFd8IiwyK5MZjE8UpbgokkgtcReH88Abaw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.12.1.tgz", + "integrity": "sha512-GoLDUi6U9ZLzlSda2Df++VSqDJg3CG+dR0+iWsv6XRw1rEq+zwt4DirM9yrxW6XWaTpmai1cWJLMfM8qQJf+yw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-json-strings": "^7.8.0" } }, - "@babel/plugin-proposal-logical-assignment-operators": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.11.0.tgz", - "integrity": "sha512-/f8p4z+Auz0Uaf+i8Ekf1iM7wUNLcViFUGiPxKeXvxTSl63B875YPiVdUDdem7hREcI0E0kSpEhS8tF5RphK7Q==", - "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" - } - }, "@babel/plugin-proposal-nullish-coalescing-operator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.10.4.tgz", - "integrity": "sha512-wq5n1M3ZUlHl9sqT2ok1T2/MTt6AXE0e1Lz4WzWBr95LsAZ5qDXe4KnFuauYyEyLiohvXFMdbsOTMyLZs91Zlw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.12.1.tgz", + "integrity": "sha512-nZY0ESiaQDI1y96+jk6VxMOaL4LPo/QDHBqL+SF3/vl6dHkTwHlOI8L4ZwuRBHgakRBw5zsVylel7QPbbGuYgg==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0" } }, "@babel/plugin-proposal-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.10.4.tgz", - "integrity": "sha512-73/G7QoRoeNkLZFxsoCCvlg4ezE4eM+57PnOqgaPOozd5myfj7p0muD1mRVJvbUWbOzD+q3No2bWbaKy+DJ8DA==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.12.7.tgz", + "integrity": "sha512-8c+uy0qmnRTeukiGsjLGy6uVs/TFjJchGXUeBqlG4VWYOdJWkhhVPdQ3uHwbmalfJwv2JsV0qffXP4asRfL2SQ==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-numeric-separator": "^7.10.4" } }, "@babel/plugin-proposal-object-rest-spread": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.11.0.tgz", - "integrity": "sha512-wzch41N4yztwoRw0ak+37wxwJM2oiIiy6huGCoqkvSTA9acYWcPfn9Y4aJqmFFJ70KTJUu29f3DQ43uJ9HXzEA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz", + "integrity": "sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-object-rest-spread": "^7.8.0", - "@babel/plugin-transform-parameters": "^7.10.4" + "@babel/plugin-transform-parameters": "^7.12.1" } }, "@babel/plugin-proposal-optional-catch-binding": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.10.4.tgz", - "integrity": "sha512-LflT6nPh+GK2MnFiKDyLiqSqVHkQnVf7hdoAvyTnnKj9xB3docGRsdPuxp6qqqW19ifK3xgc9U5/FwrSaCNX5g==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.12.1.tgz", + "integrity": "sha512-hFvIjgprh9mMw5v42sJWLI1lzU5L2sznP805zeT6rySVRA0Y18StRhDqhSxlap0oVgItRsB6WSROp4YnJTJz0g==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", "@babel/plugin-syntax-optional-catch-binding": "^7.8.0" } }, "@babel/plugin-proposal-optional-chaining": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.11.0.tgz", - "integrity": "sha512-v9fZIu3Y8562RRwhm1BbMRxtqZNFmFA2EG+pT2diuU8PT3H6T/KXoZ54KgYisfOFZHV6PfvAiBIZ9Rcz+/JCxA==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.12.7.tgz", + "integrity": "sha512-4ovylXZ0PWmwoOvhU2vhnzVNnm88/Sm9nx7V8BPgMvAzn5zDou3/Awy0EjglyubVHasJj+XCEkr/r1X3P5elCA==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-skip-transparent-expression-wrappers": "^7.11.0", + "@babel/helper-skip-transparent-expression-wrappers": "^7.12.1", "@babel/plugin-syntax-optional-chaining": "^7.8.0" } }, - "@babel/plugin-proposal-private-methods": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.10.4.tgz", - "integrity": "sha512-wh5GJleuI8k3emgTg5KkJK6kHNsGEr0uBTDBuQUBJwckk9xs1ez79ioheEVVxMLyPscB0LfkbVHslQqIzWV6Bw==", - "requires": { - "@babel/helper-create-class-features-plugin": "^7.10.4", - "@babel/helper-plugin-utils": "^7.10.4" - } - }, "@babel/plugin-proposal-unicode-property-regex": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.10.4.tgz", - "integrity": "sha512-H+3fOgPnEXFL9zGYtKQe4IDOPKYlZdF1kqFDQRRb8PK4B8af1vAGK04tF5iQAAsui+mHNBQSAtd2/ndEDe9wuA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.12.1.tgz", + "integrity": "sha512-MYq+l+PvHuw/rKUz1at/vb6nCnQ2gmJBNaM62z0OgH7B2W1D9pvkpYtlti9bGtizNIU1K3zm4bZF9F91efVY0w==", "requires": { - "@babel/helper-create-regexp-features-plugin": "^7.10.4", + "@babel/helper-create-regexp-features-plugin": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4" } }, @@ -514,14 +18629,6 @@ "@babel/helper-plugin-utils": "^7.8.0" } }, - "@babel/plugin-syntax-class-properties": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.10.4.tgz", - "integrity": "sha512-GCSBF7iUle6rNugfURwNmCGG3Z/2+opxAMLs1nND4bhEG5PuxTIggDBoeYYSujAlLtsupzOHYJQgPS3pivwXIA==", - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" - } - }, "@babel/plugin-syntax-decorators": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.10.4.tgz", @@ -538,14 +18645,6 @@ "@babel/helper-plugin-utils": "^7.8.0" } }, - "@babel/plugin-syntax-export-namespace-from": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", - "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", - "requires": { - "@babel/helper-plugin-utils": "^7.8.3" - } - }, "@babel/plugin-syntax-flow": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.10.4.tgz", @@ -563,17 +18662,9 @@ } }, "@babel/plugin-syntax-jsx": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.10.4.tgz", - "integrity": "sha512-KCg9mio9jwiARCB7WAcQ7Y1q+qicILjoK8LP/VkPkEKaf5dkaZZK1EcTe91a3JJlZ3qy6L5s9X52boEYi8DM9g==", - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" - } - }, - "@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz", + "integrity": "sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } @@ -619,9 +18710,9 @@ } }, "@babel/plugin-syntax-top-level-await": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.10.4.tgz", - "integrity": "sha512-ni1brg4lXEmWyafKr0ccFWkJG0CeMt4WV1oyeBW6EFObF4oOHclbkj5cARxAPQyAQ2UTuplJyK4nfkXIMMFvsQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz", + "integrity": "sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } @@ -635,91 +18726,91 @@ } }, "@babel/plugin-transform-arrow-functions": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.10.4.tgz", - "integrity": "sha512-9J/oD1jV0ZCBcgnoFWFq1vJd4msoKb/TCpGNFyyLt0zABdcvgK3aYikZ8HjzB14c26bc7E3Q1yugpwGy2aTPNA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.12.1.tgz", + "integrity": "sha512-5QB50qyN44fzzz4/qxDPQMBCTHgxg3n0xRBLJUmBlLoU/sFvxVWGZF/ZUfMVDQuJUKXaBhbupxIzIfZ6Fwk/0A==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-async-to-generator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.10.4.tgz", - "integrity": "sha512-F6nREOan7J5UXTLsDsZG3DXmZSVofr2tGNwfdrVwkDWHfQckbQXnXSPfD7iO+c/2HGqycwyLST3DnZ16n+cBJQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.12.1.tgz", + "integrity": "sha512-SDtqoEcarK1DFlRJ1hHRY5HvJUj5kX4qmtpMAm2QnhOlyuMC4TMdCRgW6WXpv93rZeYNeLP22y8Aq2dbcDRM1A==", "requires": { - "@babel/helper-module-imports": "^7.10.4", + "@babel/helper-module-imports": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-remap-async-to-generator": "^7.10.4" + "@babel/helper-remap-async-to-generator": "^7.12.1" } }, "@babel/plugin-transform-block-scoped-functions": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.10.4.tgz", - "integrity": "sha512-WzXDarQXYYfjaV1szJvN3AD7rZgZzC1JtjJZ8dMHUyiK8mxPRahynp14zzNjU3VkPqPsO38CzxiWO1c9ARZ8JA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.12.1.tgz", + "integrity": "sha512-5OpxfuYnSgPalRpo8EWGPzIYf0lHBWORCkj5M0oLBwHdlux9Ri36QqGW3/LR13RSVOAoUUMzoPI/jpE4ABcHoA==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-block-scoping": { - "version": "7.11.1", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.11.1.tgz", - "integrity": "sha512-00dYeDE0EVEHuuM+26+0w/SCL0BH2Qy7LwHuI4Hi4MH5gkC8/AqMN5uWFJIsoXZrAphiMm1iXzBw6L2T+eA0ew==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.12.1.tgz", + "integrity": "sha512-zJyAC9sZdE60r1nVQHblcfCj29Dh2Y0DOvlMkcqSo0ckqjiCwNiUezUKw+RjOCwGfpLRwnAeQ2XlLpsnGkvv9w==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-classes": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.10.4.tgz", - "integrity": "sha512-2oZ9qLjt161dn1ZE0Ms66xBncQH4In8Sqw1YWgBUZuGVJJS5c0OFZXL6dP2MRHrkU/eKhWg8CzFJhRQl50rQxA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.1.tgz", + "integrity": "sha512-/74xkA7bVdzQTBeSUhLLJgYIcxw/dpEpCdRDiHgPJ3Mv6uC11UhjpOhl72CgqbBCmt1qtssCyB2xnJm1+PFjog==", "requires": { "@babel/helper-annotate-as-pure": "^7.10.4", "@babel/helper-define-map": "^7.10.4", "@babel/helper-function-name": "^7.10.4", "@babel/helper-optimise-call-expression": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-replace-supers": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", "@babel/helper-split-export-declaration": "^7.10.4", "globals": "^11.1.0" } }, "@babel/plugin-transform-computed-properties": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.10.4.tgz", - "integrity": "sha512-JFwVDXcP/hM/TbyzGq3l/XWGut7p46Z3QvqFMXTfk6/09m7xZHJUN9xHfsv7vqqD4YnfI5ueYdSJtXqqBLyjBw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.12.1.tgz", + "integrity": "sha512-vVUOYpPWB7BkgUWPo4C44mUQHpTZXakEqFjbv8rQMg7TC6S6ZhGZ3otQcRH6u7+adSlE5i0sp63eMC/XGffrzg==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-destructuring": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.10.4.tgz", - "integrity": "sha512-+WmfvyfsyF603iPa6825mq6Qrb7uLjTOsa3XOFzlYcYDHSS4QmpOWOL0NNBY5qMbvrcf3tq0Cw+v4lxswOBpgA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.1.tgz", + "integrity": "sha512-fRMYFKuzi/rSiYb2uRLiUENJOKq4Gnl+6qOv5f8z0TZXg3llUwUhsNNwrwaT/6dUhJTzNpBr+CUvEWBtfNY1cw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-dotall-regex": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.10.4.tgz", - "integrity": "sha512-ZEAVvUTCMlMFAbASYSVQoxIbHm2OkG2MseW6bV2JjIygOjdVv8tuxrCTzj1+Rynh7ODb8GivUy7dzEXzEhuPaA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.12.1.tgz", + "integrity": "sha512-B2pXeRKoLszfEW7J4Hg9LoFaWEbr/kzo3teWHmtFCszjRNa/b40f9mfeqZsIDLLt/FjwQ6pz/Gdlwy85xNckBA==", "requires": { - "@babel/helper-create-regexp-features-plugin": "^7.10.4", + "@babel/helper-create-regexp-features-plugin": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-duplicate-keys": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.10.4.tgz", - "integrity": "sha512-GL0/fJnmgMclHiBTTWXNlYjYsA7rDrtsazHG6mglaGSTh0KsrW04qml+Bbz9FL0LcJIRwBWL5ZqlNHKTkU3xAA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.12.1.tgz", + "integrity": "sha512-iRght0T0HztAb/CazveUpUQrZY+aGKKaWXMJ4uf9YJtqxSUe09j3wteztCUDRHs+SRAL7yMuFqUsLoAKKzgXjw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-exponentiation-operator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.10.4.tgz", - "integrity": "sha512-S5HgLVgkBcRdyQAHbKj+7KyuWx8C6t5oETmUuwz1pt3WTWJhsUV0WIIXuVvfXMxl/QQyHKlSCNNtaIamG8fysw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.12.1.tgz", + "integrity": "sha512-7tqwy2bv48q+c1EHbXK0Zx3KXd2RVQp6OC7PbwFNt/dPTAV3Lu5sWtWuAj8owr5wqtWnqHfl2/mJlUmqkChKug==", "requires": { "@babel/helper-builder-binary-assignment-operator-visitor": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4" @@ -735,117 +18826,117 @@ } }, "@babel/plugin-transform-for-of": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.10.4.tgz", - "integrity": "sha512-ItdQfAzu9AlEqmusA/65TqJ79eRcgGmpPPFvBnGILXZH975G0LNjP1yjHvGgfuCxqrPPueXOPe+FsvxmxKiHHQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.12.1.tgz", + "integrity": "sha512-Zaeq10naAsuHo7heQvyV0ptj4dlZJwZgNAtBYBnu5nNKJoW62m0zKcIEyVECrUKErkUkg6ajMy4ZfnVZciSBhg==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-function-name": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.10.4.tgz", - "integrity": "sha512-OcDCq2y5+E0dVD5MagT5X+yTRbcvFjDI2ZVAottGH6tzqjx/LKpgkUepu3hp/u4tZBzxxpNGwLsAvGBvQ2mJzg==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.12.1.tgz", + "integrity": "sha512-JF3UgJUILoFrFMEnOJLJkRHSk6LUSXLmEFsA23aR2O5CSLUxbeUX1IZ1YQ7Sn0aXb601Ncwjx73a+FVqgcljVw==", "requires": { "@babel/helper-function-name": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-literals": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.10.4.tgz", - "integrity": "sha512-Xd/dFSTEVuUWnyZiMu76/InZxLTYilOSr1UlHV+p115Z/Le2Fi1KXkJUYz0b42DfndostYlPub3m8ZTQlMaiqQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.12.1.tgz", + "integrity": "sha512-+PxVGA+2Ag6uGgL0A5f+9rklOnnMccwEBzwYFL3EUaKuiyVnUipyXncFcfjSkbimLrODoqki1U9XxZzTvfN7IQ==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-member-expression-literals": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.10.4.tgz", - "integrity": "sha512-0bFOvPyAoTBhtcJLr9VcwZqKmSjFml1iVxvPL0ReomGU53CX53HsM4h2SzckNdkQcHox1bpAqzxBI1Y09LlBSw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.12.1.tgz", + "integrity": "sha512-1sxePl6z9ad0gFMB9KqmYofk34flq62aqMt9NqliS/7hPEpURUCMbyHXrMPlo282iY7nAvUB1aQd5mg79UD9Jg==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-modules-amd": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.10.5.tgz", - "integrity": "sha512-elm5uruNio7CTLFItVC/rIzKLfQ17+fX7EVz5W0TMgIHFo1zY0Ozzx+lgwhL4plzl8OzVn6Qasx5DeEFyoNiRw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.12.1.tgz", + "integrity": "sha512-tDW8hMkzad5oDtzsB70HIQQRBiTKrhfgwC/KkJeGsaNFTdWhKNt/BiE8c5yj19XiGyrxpbkOfH87qkNg1YGlOQ==", "requires": { - "@babel/helper-module-transforms": "^7.10.5", + "@babel/helper-module-transforms": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4", "babel-plugin-dynamic-import-node": "^2.3.3" } }, "@babel/plugin-transform-modules-commonjs": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.10.4.tgz", - "integrity": "sha512-Xj7Uq5o80HDLlW64rVfDBhao6OX89HKUmb+9vWYaLXBZOma4gA6tw4Ni1O5qVDoZWUV0fxMYA0aYzOawz0l+1w==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.12.1.tgz", + "integrity": "sha512-dY789wq6l0uLY8py9c1B48V8mVL5gZh/+PQ5ZPrylPYsnAvnEMjqsUXkuoDVPeVK+0VyGar+D08107LzDQ6pag==", "requires": { - "@babel/helper-module-transforms": "^7.10.4", + "@babel/helper-module-transforms": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-simple-access": "^7.10.4", + "@babel/helper-simple-access": "^7.12.1", "babel-plugin-dynamic-import-node": "^2.3.3" } }, "@babel/plugin-transform-modules-systemjs": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.10.5.tgz", - "integrity": "sha512-f4RLO/OL14/FP1AEbcsWMzpbUz6tssRaeQg11RH1BP/XnPpRoVwgeYViMFacnkaw4k4wjRSjn3ip1Uw9TaXuMw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.12.1.tgz", + "integrity": "sha512-Hn7cVvOavVh8yvW6fLwveFqSnd7rbQN3zJvoPNyNaQSvgfKmDBO9U1YL9+PCXGRlZD9tNdWTy5ACKqMuzyn32Q==", "requires": { "@babel/helper-hoist-variables": "^7.10.4", - "@babel/helper-module-transforms": "^7.10.5", + "@babel/helper-module-transforms": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-validator-identifier": "^7.10.4", "babel-plugin-dynamic-import-node": "^2.3.3" } }, "@babel/plugin-transform-modules-umd": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.10.4.tgz", - "integrity": "sha512-mohW5q3uAEt8T45YT7Qc5ws6mWgJAaL/8BfWD9Dodo1A3RKWli8wTS+WiQ/knF+tXlPirW/1/MqzzGfCExKECA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.12.1.tgz", + "integrity": "sha512-aEIubCS0KHKM0zUos5fIoQm+AZUMt1ZvMpqz0/H5qAQ7vWylr9+PLYurT+Ic7ID/bKLd4q8hDovaG3Zch2uz5Q==", "requires": { - "@babel/helper-module-transforms": "^7.10.4", + "@babel/helper-module-transforms": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-named-capturing-groups-regex": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.10.4.tgz", - "integrity": "sha512-V6LuOnD31kTkxQPhKiVYzYC/Jgdq53irJC/xBSmqcNcqFGV+PER4l6rU5SH2Vl7bH9mLDHcc0+l9HUOe4RNGKA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.1.tgz", + "integrity": "sha512-tB43uQ62RHcoDp9v2Nsf+dSM8sbNodbEicbQNA53zHz8pWUhsgHSJCGpt7daXxRydjb0KnfmB+ChXOv3oADp1Q==", "requires": { - "@babel/helper-create-regexp-features-plugin": "^7.10.4" + "@babel/helper-create-regexp-features-plugin": "^7.12.1" } }, "@babel/plugin-transform-new-target": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.10.4.tgz", - "integrity": "sha512-YXwWUDAH/J6dlfwqlWsztI2Puz1NtUAubXhOPLQ5gjR/qmQ5U96DY4FQO8At33JN4XPBhrjB8I4eMmLROjjLjw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.1.tgz", + "integrity": "sha512-+eW/VLcUL5L9IvJH7rT1sT0CzkdUTvPrXC2PXTn/7z7tXLBuKvezYbGdxD5WMRoyvyaujOq2fWoKl869heKjhw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-object-super": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.10.4.tgz", - "integrity": "sha512-5iTw0JkdRdJvr7sY0vHqTpnruUpTea32JHmq/atIWqsnNussbRzjEDyWep8UNztt1B5IusBYg8Irb0bLbiEBCQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.12.1.tgz", + "integrity": "sha512-AvypiGJH9hsquNUn+RXVcBdeE3KHPZexWRdimhuV59cSoOt5kFBmqlByorAeUlGG2CJWd0U+4ZtNKga/TB0cAw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-replace-supers": "^7.10.4" + "@babel/helper-replace-supers": "^7.12.1" } }, "@babel/plugin-transform-parameters": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.10.5.tgz", - "integrity": "sha512-xPHwUj5RdFV8l1wuYiu5S9fqWGM2DrYc24TMvUiRrPVm+SM3XeqU9BcokQX/kEUe+p2RBwy+yoiR1w/Blq6ubw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.1.tgz", + "integrity": "sha512-xq9C5EQhdPK23ZeCdMxl8bbRnAgHFrw5EOC3KJUsSylZqdkCaFEXxGSBuTSObOpiiHHNyb82es8M1QYgfQGfNg==", "requires": { - "@babel/helper-get-function-arity": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-property-literals": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.10.4.tgz", - "integrity": "sha512-ofsAcKiUxQ8TY4sScgsGeR2vJIsfrzqvFb9GvJ5UdXDzl+MyYCaBj/FGzXuv7qE0aJcjWMILny1epqelnFlz8g==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.12.1.tgz", + "integrity": "sha512-6MTCR/mZ1MQS+AwZLplX4cEySjCpnIF26ToWo942nqn8hXSm7McaHQNeGx/pt7suI1TWOWMfa/NgBhiqSnX0cQ==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } @@ -859,73 +18950,62 @@ } }, "@babel/plugin-transform-react-display-name": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.10.4.tgz", - "integrity": "sha512-Zd4X54Mu9SBfPGnEcaGcOrVAYOtjT2on8QZkLKEq1S/tHexG39d9XXGZv19VfRrDjPJzFmPfTAqOQS1pfFOujw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.12.1.tgz", + "integrity": "sha512-cAzB+UzBIrekfYxyLlFqf/OagTvHLcVBb5vpouzkYkBclRPraiygVnafvAoipErZLI8ANv8Ecn6E/m5qPXD26w==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-react-jsx": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.10.4.tgz", - "integrity": "sha512-L+MfRhWjX0eI7Js093MM6MacKU4M6dnCRa/QPDwYMxjljzSCzzlzKzj9Pk4P3OtrPcxr2N3znR419nr3Xw+65A==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.12.7.tgz", + "integrity": "sha512-YFlTi6MEsclFAPIDNZYiCRbneg1MFGao9pPG9uD5htwE0vDbPaMUMeYd6itWjw7K4kro4UbdQf3ljmFl9y48dQ==", "requires": { "@babel/helper-builder-react-jsx": "^7.10.4", - "@babel/helper-builder-react-jsx-experimental": "^7.10.4", + "@babel/helper-builder-react-jsx-experimental": "^7.12.4", "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-jsx": "^7.10.4" + "@babel/plugin-syntax-jsx": "^7.12.1" } }, "@babel/plugin-transform-react-jsx-development": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.10.4.tgz", - "integrity": "sha512-RM3ZAd1sU1iQ7rI2dhrZRZGv0aqzNQMbkIUCS1txYpi9wHQ2ZHNjo5TwX+UD6pvFW4AbWqLVYvKy5qJSAyRGjQ==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.12.7.tgz", + "integrity": "sha512-Rs3ETtMtR3VLXFeYRChle5SsP/P9Jp/6dsewBQfokDSzKJThlsuFcnzLTDRALiUmTC48ej19YD9uN1mupEeEDg==", "requires": { - "@babel/helper-builder-react-jsx-experimental": "^7.10.4", + "@babel/helper-builder-react-jsx-experimental": "^7.12.4", "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-jsx": "^7.10.4" + "@babel/plugin-syntax-jsx": "^7.12.1" } }, "@babel/plugin-transform-react-jsx-self": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.10.4.tgz", - "integrity": "sha512-yOvxY2pDiVJi0axdTWHSMi5T0DILN+H+SaeJeACHKjQLezEzhLx9nEF9xgpBLPtkZsks9cnb5P9iBEi21En3gg==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.12.1.tgz", + "integrity": "sha512-FbpL0ieNWiiBB5tCldX17EtXgmzeEZjFrix72rQYeq9X6nUK38HCaxexzVQrZWXanxKJPKVVIU37gFjEQYkPkA==", "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-jsx": "^7.10.4" + "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-react-jsx-source": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.10.5.tgz", - "integrity": "sha512-wTeqHVkN1lfPLubRiZH3o73f4rfon42HpgxUSs86Nc+8QIcm/B9s8NNVXu/gwGcOyd7yDib9ikxoDLxJP0UiDA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.12.1.tgz", + "integrity": "sha512-keQ5kBfjJNRc6zZN1/nVHCd6LLIHq4aUKcVnvE/2l+ZZROSbqoiGFRtT5t3Is89XJxBQaP7NLZX2jgGHdZvvFQ==", "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-syntax-jsx": "^7.10.4" - } - }, - "@babel/plugin-transform-react-pure-annotations": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.10.4.tgz", - "integrity": "sha512-+njZkqcOuS8RaPakrnR9KvxjoG1ASJWpoIv/doyWngId88JoFlPlISenGXjrVacZUIALGUr6eodRs1vmPnF23A==", - "requires": { - "@babel/helper-annotate-as-pure": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-regenerator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.10.4.tgz", - "integrity": "sha512-3thAHwtor39A7C04XucbMg17RcZ3Qppfxr22wYzZNcVIkPHfpM9J0SO8zuCV6SZa265kxBJSrfKTvDCYqBFXGw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.1.tgz", + "integrity": "sha512-gYrHqs5itw6i4PflFX3OdBPMQdPbF4bj2REIUxlMRUFk0/ZOAIpDFuViuxPjUL7YC8UPnf+XG7/utJvqXdPKng==", "requires": { "regenerator-transform": "^0.14.2" } }, "@babel/plugin-transform-reserved-words": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.10.4.tgz", - "integrity": "sha512-hGsw1O6Rew1fkFbDImZIEqA8GoidwTAilwCyWqLBM9f+e/u/sQMQu7uX6dyokfOayRuuVfKOW4O7HvaBWM+JlQ==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.12.1.tgz", + "integrity": "sha512-pOnUfhyPKvZpVyBHhSBoX8vfA09b7r00Pmm1sH+29ae2hMTKVmSp4Ztsr8KBKjLjx17H0eJqaRC3bR2iThM54A==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } @@ -949,44 +19029,42 @@ } }, "@babel/plugin-transform-shorthand-properties": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.10.4.tgz", - "integrity": "sha512-AC2K/t7o07KeTIxMoHneyX90v3zkm5cjHJEokrPEAGEy3UCp8sLKfnfOIGdZ194fyN4wfX/zZUWT9trJZ0qc+Q==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.12.1.tgz", + "integrity": "sha512-GFZS3c/MhX1OusqB1MZ1ct2xRzX5ppQh2JU1h2Pnfk88HtFTM+TWQqJNfwkmxtPQtb/s1tk87oENfXJlx7rSDw==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-spread": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.11.0.tgz", - "integrity": "sha512-UwQYGOqIdQJe4aWNyS7noqAnN2VbaczPLiEtln+zPowRNlD+79w3oi2TWfYe0eZgd+gjZCbsydN7lzWysDt+gw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.12.1.tgz", + "integrity": "sha512-vuLp8CP0BE18zVYjsEBZ5xoCecMK6LBMMxYzJnh01rxQRvhNhH1csMMmBfNo5tGpGO+NhdSNW2mzIvBu3K1fng==", "requires": { "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-skip-transparent-expression-wrappers": "^7.11.0" + "@babel/helper-skip-transparent-expression-wrappers": "^7.12.1" } }, "@babel/plugin-transform-sticky-regex": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.10.4.tgz", - "integrity": "sha512-Ddy3QZfIbEV0VYcVtFDCjeE4xwVTJWTmUtorAJkn6u/92Z/nWJNV+mILyqHKrUxXYKA2EoCilgoPePymKL4DvQ==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.12.7.tgz", + "integrity": "sha512-VEiqZL5N/QvDbdjfYQBhruN0HYjSPjC4XkeqW4ny/jNtH9gcbgaqBIXYEZCNnESMAGs0/K/R7oFGMhOyu/eIxg==", "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/helper-regex": "^7.10.4" + "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-template-literals": { - "version": "7.10.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.10.5.tgz", - "integrity": "sha512-V/lnPGIb+KT12OQikDvgSuesRX14ck5FfJXt6+tXhdkJ+Vsd0lDCVtF6jcB4rNClYFzaB2jusZ+lNISDk2mMMw==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.12.1.tgz", + "integrity": "sha512-b4Zx3KHi+taXB1dVRBhVJtEPi9h1THCeKmae2qP0YdUHIFhVjtpqqNfxeVAa1xeHVhAy4SbHxEwx5cltAu5apw==", "requires": { - "@babel/helper-annotate-as-pure": "^7.10.4", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/plugin-transform-typeof-symbol": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.10.4.tgz", - "integrity": "sha512-QqNgYwuuW0y0H+kUE/GWSR45t/ccRhe14Fs/4ZRouNNQsyd4o3PG4OtHiIrepbM2WKUBDAXKCAK/Lk4VhzTaGA==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.12.1.tgz", + "integrity": "sha512-EPGgpGy+O5Kg5pJFNDKuxt9RdmTgj5sgrus2XVeMp/ZIbOESadgILUbm50SNpghOh3/6yrbsH+NB5+WJTmsA7Q==", "requires": { "@babel/helper-plugin-utils": "^7.10.4" } @@ -1001,92 +19079,76 @@ "@babel/plugin-syntax-typescript": "^7.10.4" } }, - "@babel/plugin-transform-unicode-escapes": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.10.4.tgz", - "integrity": "sha512-y5XJ9waMti2J+e7ij20e+aH+fho7Wb7W8rNuu72aKRwCHFqQdhkdU2lo3uZ9tQuboEJcUFayXdARhcxLQ3+6Fg==", - "requires": { - "@babel/helper-plugin-utils": "^7.10.4" - } - }, "@babel/plugin-transform-unicode-regex": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.10.4.tgz", - "integrity": "sha512-wNfsc4s8N2qnIwpO/WP2ZiSyjfpTamT2C9V9FDH/Ljub9zw6P3SjkXcFmc0RQUt96k2fmIvtla2MMjgTwIAC+A==", + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.12.1.tgz", + "integrity": "sha512-SqH4ClNngh/zGwHZOOQMTD+e8FGWexILV+ePMyiDJttAWRh5dhDL8rcl5lSgU3Huiq6Zn6pWTMvdPAb21Dwdyg==", "requires": { - "@babel/helper-create-regexp-features-plugin": "^7.10.4", + "@babel/helper-create-regexp-features-plugin": "^7.12.1", "@babel/helper-plugin-utils": "^7.10.4" } }, "@babel/preset-env": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.11.0.tgz", - "integrity": "sha512-2u1/k7rG/gTh02dylX2kL3S0IJNF+J6bfDSp4DI2Ma8QN6Y9x9pmAax59fsCk6QUQG0yqH47yJWA+u1I1LccAg==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.9.0.tgz", + "integrity": "sha512-712DeRXT6dyKAM/FMbQTV/FvRCms2hPCx+3weRjZ8iQVQWZejWWk1wwG6ViWMyqb/ouBbGOl5b6aCk0+j1NmsQ==", "requires": { - "@babel/compat-data": "^7.11.0", - "@babel/helper-compilation-targets": "^7.10.4", - "@babel/helper-module-imports": "^7.10.4", - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-proposal-async-generator-functions": "^7.10.4", - "@babel/plugin-proposal-class-properties": "^7.10.4", - "@babel/plugin-proposal-dynamic-import": "^7.10.4", - "@babel/plugin-proposal-export-namespace-from": "^7.10.4", - "@babel/plugin-proposal-json-strings": "^7.10.4", - "@babel/plugin-proposal-logical-assignment-operators": "^7.11.0", - "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4", - "@babel/plugin-proposal-numeric-separator": "^7.10.4", - "@babel/plugin-proposal-object-rest-spread": "^7.11.0", - "@babel/plugin-proposal-optional-catch-binding": "^7.10.4", - "@babel/plugin-proposal-optional-chaining": "^7.11.0", - "@babel/plugin-proposal-private-methods": "^7.10.4", - "@babel/plugin-proposal-unicode-property-regex": "^7.10.4", + "@babel/compat-data": "^7.9.0", + "@babel/helper-compilation-targets": "^7.8.7", + "@babel/helper-module-imports": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-proposal-async-generator-functions": "^7.8.3", + "@babel/plugin-proposal-dynamic-import": "^7.8.3", + "@babel/plugin-proposal-json-strings": "^7.8.3", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-proposal-numeric-separator": "^7.8.3", + "@babel/plugin-proposal-object-rest-spread": "^7.9.0", + "@babel/plugin-proposal-optional-catch-binding": "^7.8.3", + "@babel/plugin-proposal-optional-chaining": "^7.9.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.8.3", "@babel/plugin-syntax-async-generators": "^7.8.0", - "@babel/plugin-syntax-class-properties": "^7.10.4", "@babel/plugin-syntax-dynamic-import": "^7.8.0", - "@babel/plugin-syntax-export-namespace-from": "^7.8.3", "@babel/plugin-syntax-json-strings": "^7.8.0", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-numeric-separator": "^7.8.0", "@babel/plugin-syntax-object-rest-spread": "^7.8.0", "@babel/plugin-syntax-optional-catch-binding": "^7.8.0", "@babel/plugin-syntax-optional-chaining": "^7.8.0", - "@babel/plugin-syntax-top-level-await": "^7.10.4", - "@babel/plugin-transform-arrow-functions": "^7.10.4", - "@babel/plugin-transform-async-to-generator": "^7.10.4", - "@babel/plugin-transform-block-scoped-functions": "^7.10.4", - "@babel/plugin-transform-block-scoping": "^7.10.4", - "@babel/plugin-transform-classes": "^7.10.4", - "@babel/plugin-transform-computed-properties": "^7.10.4", - "@babel/plugin-transform-destructuring": "^7.10.4", - "@babel/plugin-transform-dotall-regex": "^7.10.4", - "@babel/plugin-transform-duplicate-keys": "^7.10.4", - "@babel/plugin-transform-exponentiation-operator": "^7.10.4", - "@babel/plugin-transform-for-of": "^7.10.4", - "@babel/plugin-transform-function-name": "^7.10.4", - "@babel/plugin-transform-literals": "^7.10.4", - "@babel/plugin-transform-member-expression-literals": "^7.10.4", - "@babel/plugin-transform-modules-amd": "^7.10.4", - "@babel/plugin-transform-modules-commonjs": "^7.10.4", - "@babel/plugin-transform-modules-systemjs": "^7.10.4", - "@babel/plugin-transform-modules-umd": "^7.10.4", - "@babel/plugin-transform-named-capturing-groups-regex": "^7.10.4", - "@babel/plugin-transform-new-target": "^7.10.4", - "@babel/plugin-transform-object-super": "^7.10.4", - "@babel/plugin-transform-parameters": "^7.10.4", - "@babel/plugin-transform-property-literals": "^7.10.4", - "@babel/plugin-transform-regenerator": "^7.10.4", - "@babel/plugin-transform-reserved-words": "^7.10.4", - "@babel/plugin-transform-shorthand-properties": "^7.10.4", - "@babel/plugin-transform-spread": "^7.11.0", - "@babel/plugin-transform-sticky-regex": "^7.10.4", - "@babel/plugin-transform-template-literals": "^7.10.4", - "@babel/plugin-transform-typeof-symbol": "^7.10.4", - "@babel/plugin-transform-unicode-escapes": "^7.10.4", - "@babel/plugin-transform-unicode-regex": "^7.10.4", + "@babel/plugin-syntax-top-level-await": "^7.8.3", + "@babel/plugin-transform-arrow-functions": "^7.8.3", + "@babel/plugin-transform-async-to-generator": "^7.8.3", + "@babel/plugin-transform-block-scoped-functions": "^7.8.3", + "@babel/plugin-transform-block-scoping": "^7.8.3", + "@babel/plugin-transform-classes": "^7.9.0", + "@babel/plugin-transform-computed-properties": "^7.8.3", + "@babel/plugin-transform-destructuring": "^7.8.3", + "@babel/plugin-transform-dotall-regex": "^7.8.3", + "@babel/plugin-transform-duplicate-keys": "^7.8.3", + "@babel/plugin-transform-exponentiation-operator": "^7.8.3", + "@babel/plugin-transform-for-of": "^7.9.0", + "@babel/plugin-transform-function-name": "^7.8.3", + "@babel/plugin-transform-literals": "^7.8.3", + "@babel/plugin-transform-member-expression-literals": "^7.8.3", + "@babel/plugin-transform-modules-amd": "^7.9.0", + "@babel/plugin-transform-modules-commonjs": "^7.9.0", + "@babel/plugin-transform-modules-systemjs": "^7.9.0", + "@babel/plugin-transform-modules-umd": "^7.9.0", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.8.3", + "@babel/plugin-transform-new-target": "^7.8.3", + "@babel/plugin-transform-object-super": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.8.7", + "@babel/plugin-transform-property-literals": "^7.8.3", + "@babel/plugin-transform-regenerator": "^7.8.7", + "@babel/plugin-transform-reserved-words": "^7.8.3", + "@babel/plugin-transform-shorthand-properties": "^7.8.3", + "@babel/plugin-transform-spread": "^7.8.3", + "@babel/plugin-transform-sticky-regex": "^7.8.3", + "@babel/plugin-transform-template-literals": "^7.8.3", + "@babel/plugin-transform-typeof-symbol": "^7.8.4", + "@babel/plugin-transform-unicode-regex": "^7.8.3", "@babel/preset-modules": "^0.1.3", - "@babel/types": "^7.11.0", - "browserslist": "^4.12.0", + "@babel/types": "^7.9.0", + "browserslist": "^4.9.1", "core-js-compat": "^3.6.2", "invariant": "^2.2.2", "levenary": "^1.1.1", @@ -1113,17 +19175,16 @@ } }, "@babel/preset-react": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.10.4.tgz", - "integrity": "sha512-BrHp4TgOIy4M19JAfO1LhycVXOPWdDbTRep7eVyatf174Hff+6Uk53sDyajqZPu8W1qXRBiYOfIamek6jA7YVw==", + "version": "7.9.1", + "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.9.1.tgz", + "integrity": "sha512-aJBYF23MPj0RNdp/4bHnAP0NVqqZRr9kl0NAOP4nJCex6OYVio59+dnQzsAWFuogdLyeaKA1hmfUIVZkY5J+TQ==", "requires": { - "@babel/helper-plugin-utils": "^7.10.4", - "@babel/plugin-transform-react-display-name": "^7.10.4", - "@babel/plugin-transform-react-jsx": "^7.10.4", - "@babel/plugin-transform-react-jsx-development": "^7.10.4", - "@babel/plugin-transform-react-jsx-self": "^7.10.4", - "@babel/plugin-transform-react-jsx-source": "^7.10.4", - "@babel/plugin-transform-react-pure-annotations": "^7.10.4" + "@babel/helper-plugin-utils": "^7.8.3", + "@babel/plugin-transform-react-display-name": "^7.8.3", + "@babel/plugin-transform-react-jsx": "^7.9.1", + "@babel/plugin-transform-react-jsx-development": "^7.9.0", + "@babel/plugin-transform-react-jsx-self": "^7.9.0", + "@babel/plugin-transform-react-jsx-source": "^7.9.0" } }, "@babel/preset-typescript": { @@ -1136,9 +19197,9 @@ } }, "@babel/runtime": { - "version": "7.9.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.2.tgz", - "integrity": "sha512-NE2DtOdufG7R5vnfQUTehdTfNycfUANEtCa9PssN9O/xmTzP4E08UI797ixaei6hBEVL9BI/PsdJS5x7mWoB9Q==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.0.tgz", + "integrity": "sha512-cTIudHnzuWLS56ik4DnRnqqNf8MkdUzV4iFFI1h7Jo9xvrpQROYaAnaSd2mHLQAzzZAPfATynX5ord6YlNYNMA==", "requires": { "regenerator-runtime": "^0.13.4" } @@ -1163,25 +19224,25 @@ } }, "@babel/traverse": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.11.0.tgz", - "integrity": "sha512-ZB2V+LskoWKNpMq6E5UUCrjtDUh5IOTAyIl0dTjIEoXum/iKWkoIEKIRDnUucO6f+2FzNkE0oD4RLKoPIufDtg==", + "version": "7.12.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.12.9.tgz", + "integrity": "sha512-iX9ajqnLdoU1s1nHt36JDI9KG4k+vmI8WgjK5d+aDTwQbL2fUnzedNedssA645Ede3PM2ma1n8Q4h2ohwXgMXw==", "requires": { "@babel/code-frame": "^7.10.4", - "@babel/generator": "^7.11.0", + "@babel/generator": "^7.12.5", "@babel/helper-function-name": "^7.10.4", "@babel/helper-split-export-declaration": "^7.11.0", - "@babel/parser": "^7.11.0", - "@babel/types": "^7.11.0", + "@babel/parser": "^7.12.7", + "@babel/types": "^7.12.7", "debug": "^4.1.0", "globals": "^11.1.0", "lodash": "^4.17.19" } }, "@babel/types": { - "version": "7.11.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.11.0.tgz", - "integrity": "sha512-O53yME4ZZI0jO1EVGtF1ePGl0LHirG4P1ibcD80XyzZcKhcMFeCXmh4Xb1ifGBIV233Qg12x4rBfQgA+tmOukA==", + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.7.tgz", + "integrity": "sha512-MNyI92qZq6jrQkXvtIiykvl4WtoRrVV9MPn+ZfsoEENjiWcBQ3ZSHrkxnJWgWtLX3XXqX5hrSQ+X69wkmesXuQ==", "requires": { "@babel/helper-validator-identifier": "^7.10.4", "lodash": "^4.17.19", @@ -1337,24 +19398,6 @@ "strip-ansi": "^5.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-escapes": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", @@ -1415,26 +19458,6 @@ "@jest/transform": "^24.9.0", "@jest/types": "^24.9.0", "jest-mock": "^24.9.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "@jest/fake-timers": { @@ -1445,26 +19468,6 @@ "@jest/types": "^24.9.0", "jest-message-util": "^24.9.0", "jest-mock": "^24.9.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "@jest/reporters": { @@ -1495,24 +19498,6 @@ "string-length": "^2.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -1594,26 +19579,6 @@ "@jest/console": "^24.9.0", "@jest/types": "^24.9.0", "@types/istanbul-lib-coverage": "^2.0.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "@jest/test-sequencer": { @@ -1650,24 +19615,6 @@ "write-file-atomic": "2.4.1" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -1720,14 +19667,13 @@ } }, "@jest/types": { - "version": "25.2.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-25.2.3.tgz", - "integrity": "sha512-6oLQwO9mKif3Uph3RX5J1i3S7X7xtDHWBaaaoeKw8hOzV6YUd0qDcYcHZ6QXMHDIzSr7zzrEa51o2Ovlj6AtKQ==", + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", + "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", "requires": { "@types/istanbul-lib-coverage": "^2.0.0", "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^15.0.0", - "chalk": "^3.0.0" + "@types/yargs": "^13.0.0" } }, "@material-ui/core": { @@ -1994,11 +19940,6 @@ "resolved": "https://registry.npmjs.org/@types/classnames/-/classnames-2.2.10.tgz", "integrity": "sha512-1UzDldn9GfYYEsWWnn/P4wkTlkZDH7lDb0wBMGbtIQc9zXEQq7FlKBdZUn6OBqD8sKZZ2RQO2mAjGpXiDGoRmQ==" }, - "@types/color-name": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", - "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" - }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", @@ -2149,9 +20090,9 @@ "integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==" }, "@types/yargs": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.4.tgz", - "integrity": "sha512-9T1auFmbPZoxHz0enUFlUuKRy3it01R+hlggyVUMtnCTQRunsQYifnSGb8hET4Xo8yiC0o0r1paW3ud5+rbURg==", + "version": "13.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.11.tgz", + "integrity": "sha512-NRqD6T4gktUrDi1o1wLH3EKC1o2caCr7/wR87ODcbVITQF106OM3sFN92ysZ++wqelOd1CTzatnOBRDYYG6wGQ==", "requires": { "@types/yargs-parser": "*" } @@ -2208,10 +20149,21 @@ "tsutils": "^3.17.1" }, "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, "semver": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", - "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", + "requires": { + "lru-cache": "^6.0.0" + } } } }, @@ -2398,9 +20350,9 @@ } }, "acorn": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", - "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==" + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==" }, "acorn-globals": { "version": "4.3.4", @@ -2412,9 +20364,9 @@ }, "dependencies": { "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" } } }, @@ -2434,43 +20386,22 @@ "integrity": "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==" }, "adjust-sourcemap-loader": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-2.0.0.tgz", - "integrity": "sha512-4hFsTsn58+YjrU9qKzML2JSSDqKvN8mUGQ0nNIrfPi8hmIONT4L3uUaT6MKdMsZ9AjsU6D2xDkZxCkbQPxChrA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-3.0.0.tgz", + "integrity": "sha512-YBrGyT2/uVQ/c6Rr+t6ZJXniY03YtHGMJQYal368burRGYKqhx9qGTWqcBU5s1CwYY9E/ri63RYyG1IacMZtqw==", "requires": { - "assert": "1.4.1", - "camelcase": "5.0.0", - "loader-utils": "1.2.3", - "object-path": "0.11.4", - "regex-parser": "2.2.10" + "loader-utils": "^2.0.0", + "regex-parser": "^2.2.11" }, "dependencies": { - "camelcase": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.0.0.tgz", - "integrity": "sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==" - }, - "emojis-list": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz", - "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=" - }, - "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", - "requires": { - "minimist": "^1.2.0" - } - }, "loader-utils": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz", - "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", "requires": { "big.js": "^5.2.2", - "emojis-list": "^2.0.0", - "json5": "^1.0.1" + "emojis-list": "^3.0.0", + "json5": "^2.1.2" } } } @@ -2541,11 +20472,10 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" }, "ansi-styles": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", - "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "requires": { - "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" } }, @@ -2855,24 +20785,6 @@ "slash": "^2.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -2988,9 +20900,9 @@ } }, "import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", "requires": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -3117,99 +21029,6 @@ "requires": { "@babel/helper-plugin-utils": "^7.8.3" } - }, - "@babel/preset-env": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.9.0.tgz", - "integrity": "sha512-712DeRXT6dyKAM/FMbQTV/FvRCms2hPCx+3weRjZ8iQVQWZejWWk1wwG6ViWMyqb/ouBbGOl5b6aCk0+j1NmsQ==", - "requires": { - "@babel/compat-data": "^7.9.0", - "@babel/helper-compilation-targets": "^7.8.7", - "@babel/helper-module-imports": "^7.8.3", - "@babel/helper-plugin-utils": "^7.8.3", - "@babel/plugin-proposal-async-generator-functions": "^7.8.3", - "@babel/plugin-proposal-dynamic-import": "^7.8.3", - "@babel/plugin-proposal-json-strings": "^7.8.3", - "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-proposal-numeric-separator": "^7.8.3", - "@babel/plugin-proposal-object-rest-spread": "^7.9.0", - "@babel/plugin-proposal-optional-catch-binding": "^7.8.3", - "@babel/plugin-proposal-optional-chaining": "^7.9.0", - "@babel/plugin-proposal-unicode-property-regex": "^7.8.3", - "@babel/plugin-syntax-async-generators": "^7.8.0", - "@babel/plugin-syntax-dynamic-import": "^7.8.0", - "@babel/plugin-syntax-json-strings": "^7.8.0", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.0", - "@babel/plugin-syntax-numeric-separator": "^7.8.0", - "@babel/plugin-syntax-object-rest-spread": "^7.8.0", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.0", - "@babel/plugin-syntax-optional-chaining": "^7.8.0", - "@babel/plugin-syntax-top-level-await": "^7.8.3", - "@babel/plugin-transform-arrow-functions": "^7.8.3", - "@babel/plugin-transform-async-to-generator": "^7.8.3", - "@babel/plugin-transform-block-scoped-functions": "^7.8.3", - "@babel/plugin-transform-block-scoping": "^7.8.3", - "@babel/plugin-transform-classes": "^7.9.0", - "@babel/plugin-transform-computed-properties": "^7.8.3", - "@babel/plugin-transform-destructuring": "^7.8.3", - "@babel/plugin-transform-dotall-regex": "^7.8.3", - "@babel/plugin-transform-duplicate-keys": "^7.8.3", - "@babel/plugin-transform-exponentiation-operator": "^7.8.3", - "@babel/plugin-transform-for-of": "^7.9.0", - "@babel/plugin-transform-function-name": "^7.8.3", - "@babel/plugin-transform-literals": "^7.8.3", - "@babel/plugin-transform-member-expression-literals": "^7.8.3", - "@babel/plugin-transform-modules-amd": "^7.9.0", - "@babel/plugin-transform-modules-commonjs": "^7.9.0", - "@babel/plugin-transform-modules-systemjs": "^7.9.0", - "@babel/plugin-transform-modules-umd": "^7.9.0", - "@babel/plugin-transform-named-capturing-groups-regex": "^7.8.3", - "@babel/plugin-transform-new-target": "^7.8.3", - "@babel/plugin-transform-object-super": "^7.8.3", - "@babel/plugin-transform-parameters": "^7.8.7", - "@babel/plugin-transform-property-literals": "^7.8.3", - "@babel/plugin-transform-regenerator": "^7.8.7", - "@babel/plugin-transform-reserved-words": "^7.8.3", - "@babel/plugin-transform-shorthand-properties": "^7.8.3", - "@babel/plugin-transform-spread": "^7.8.3", - "@babel/plugin-transform-sticky-regex": "^7.8.3", - "@babel/plugin-transform-template-literals": "^7.8.3", - "@babel/plugin-transform-typeof-symbol": "^7.8.4", - "@babel/plugin-transform-unicode-regex": "^7.8.3", - "@babel/preset-modules": "^0.1.3", - "@babel/types": "^7.9.0", - "browserslist": "^4.9.1", - "core-js-compat": "^3.6.2", - "invariant": "^2.2.2", - "levenary": "^1.1.1", - "semver": "^5.5.0" - } - }, - "@babel/preset-react": { - "version": "7.9.1", - "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.9.1.tgz", - "integrity": "sha512-aJBYF23MPj0RNdp/4bHnAP0NVqqZRr9kl0NAOP4nJCex6OYVio59+dnQzsAWFuogdLyeaKA1hmfUIVZkY5J+TQ==", - "requires": { - "@babel/helper-plugin-utils": "^7.8.3", - "@babel/plugin-transform-react-display-name": "^7.8.3", - "@babel/plugin-transform-react-jsx": "^7.9.1", - "@babel/plugin-transform-react-jsx-development": "^7.9.0", - "@babel/plugin-transform-react-jsx-self": "^7.9.0", - "@babel/plugin-transform-react-jsx-source": "^7.9.0" - } - }, - "@babel/runtime": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.9.0.tgz", - "integrity": "sha512-cTIudHnzuWLS56ik4DnRnqqNf8MkdUzV4iFFI1h7Jo9xvrpQROYaAnaSd2mHLQAzzZAPfATynX5ord6YlNYNMA==", - "requires": { - "regenerator-runtime": "^0.13.4" - } - }, - "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" } } }, @@ -3223,9 +21042,9 @@ }, "dependencies": { "core-js": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", - "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==" + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.12.tgz", + "integrity": "sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ==" }, "regenerator-runtime": { "version": "0.11.1", @@ -3549,14 +21368,15 @@ } }, "browserslist": { - "version": "4.14.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.14.0.tgz", - "integrity": "sha512-pUsXKAF2lVwhmtpeA3LJrZ76jXuusrNyhduuQs7CDFf9foT4Y38aQOserd2lMe5DSSrjf3fx34oHwryuvxAUgQ==", + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.15.0.tgz", + "integrity": "sha512-IJ1iysdMkGmjjYeRlDU8PQejVwxvVO5QOfXH7ylW31GO6LwNRSmm/SgRXtNsEXqMLl2e+2H5eEJ7sfynF8TCaQ==", "requires": { - "caniuse-lite": "^1.0.30001111", - "electron-to-chromium": "^1.3.523", - "escalade": "^3.0.2", - "node-releases": "^1.1.60" + "caniuse-lite": "^1.0.30001164", + "colorette": "^1.2.1", + "electron-to-chromium": "^1.3.612", + "escalade": "^3.1.1", + "node-releases": "^1.1.67" } }, "bser": { @@ -3660,6 +21480,15 @@ "unset-value": "^1.0.0" } }, + "call-bind": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.0.tgz", + "integrity": "sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.0" + } + }, "call-me-maybe": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz", @@ -3712,9 +21541,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001120", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001120.tgz", - "integrity": "sha512-JBP68okZs1X8D7MQTY602jxMYBmXEKOFkzTBaNSkubooMPFOAv2TXWaKle7qgHpjLDhUzA/TMT0qsNleVyXGUQ==" + "version": "1.0.30001165", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001165.tgz", + "integrity": "sha512-8cEsSMwXfx7lWSUMA2s08z9dIgsnR5NAqjXP23stdsU3AUWkCr/rr4s4OFtHXn5XXr6+7kam3QFVoYyXNPdJPA==" }, "capture-exit": { "version": "2.0.0", @@ -3749,9 +21578,9 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, "chokidar": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", - "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz", + "integrity": "sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==", "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", @@ -3760,7 +21589,7 @@ "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.4.0" + "readdirp": "~3.5.0" }, "dependencies": { "anymatch": { @@ -3890,9 +21719,9 @@ } }, "cli-width": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", - "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==" + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" }, "cliui": { "version": "5.0.0", @@ -4171,14 +22000,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -4257,16 +22078,16 @@ "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=" }, "core-js": { - "version": "3.6.5", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.5.tgz", - "integrity": "sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==" + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.8.0.tgz", + "integrity": "sha512-W2VYNB0nwQQE7tKS7HzXd7r2y/y2SVJl4ga6oH/dnaLFzM0o2lB2P3zCkWj5Wc/zyMYjtgd5Hmhk0ObkQFZOIA==" }, "core-js-compat": { - "version": "3.6.5", - "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.6.5.tgz", - "integrity": "sha512-7ItTKOhOZbznhXAQ2g/slGg1PJV5zDO/WdkTwi7UEOJmkvsE32PWvx6mKtDjiMpjnR2CNf6BAD6sSxIlv7ptng==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.8.0.tgz", + "integrity": "sha512-o9QKelQSxQMYWHXc/Gc4L8bx/4F7TTraE5rhuN8I7mKBt5dBIUpXpIR3omv70ebr8ST5R3PqbDQr+ZI3+Tt1FQ==", "requires": { - "browserslist": "^4.8.5", + "browserslist": "^4.14.7", "semver": "7.0.0" }, "dependencies": { @@ -4518,9 +22339,9 @@ } }, "css-what": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.3.0.tgz", - "integrity": "sha512-pv9JPyatiPaQ6pf4OvD/dbfm0o5LviWmwxNWzblYf/1u9QZd0ihV+PMwy5jdQWQ3349kZmKEx9WXuSka2dM4cg==" + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-3.4.2.tgz", + "integrity": "sha512-ACUm3L0/jiZTqfzRM3Hi9Q8eZqd6IK37mMWPLz9PJxkLWllYeRf+EHUSHYEtFop2Eqytaq1FizFVh7XfBnXCDQ==" }, "cssdb": { "version": "4.4.0", @@ -4646,9 +22467,9 @@ } }, "csstype": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.9.tgz", - "integrity": "sha512-xz39Sb4+OaTsULgUERcCk+TJj8ylkL4aSVDQiX/ksxbELSqwkgt4d4RD7fovIdgJGSuNYqwZEiVjYY5l0ask+Q==" + "version": "2.6.14", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-2.6.14.tgz", + "integrity": "sha512-2mSc+VEpGPblzAxyeR+vZhJKgYg0Og0nnRi7pmRXFYYxSfnOnW8A5wwQb4n4cE2nIOzqKOAzLCaEX6aBmNEv8A==" }, "cyclist": { "version": "1.0.1", @@ -4700,11 +22521,11 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "decamelize": { @@ -4898,9 +22719,9 @@ } }, "diff-sequences": { - "version": "25.2.1", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.1.tgz", - "integrity": "sha512-foe7dXnGlSh3jR1ovJmdv+77VQj98eKCHHwJPbZ2eEf0fHwKbkZicpPxEch9smZ+n2dnF6QFwkOQdLq9hpeJUg==" + "version": "25.2.6", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-25.2.6.tgz", + "integrity": "sha512-Hq8o7+6GaZeoFjtpgvRBUknSXNeJiCx7V9Fr94ZMljNiCr9n9L8H8aJqgWOQiDDGdyn29fRNcDdRVJ5fdyihfg==" }, "diffie-hellman": { "version": "5.0.3", @@ -4976,9 +22797,9 @@ }, "dependencies": { "csstype": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.2.tgz", - "integrity": "sha512-ofovWglpqoqbfLNOTBNZLSbMuGrblAf1efvvArGKOZMBrIoJeu5UsAipQolkijtyQx5MtAzT/J9IHj/CEY1mJw==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.5.tgz", + "integrity": "sha512-uVDi8LpBUKQj6sdxNaTetL6FpeCqTjOvAQuQUa/qAqq8oOd4ivkbhgnqayl0dnPal8Tb/yB1tF+gOvCBiicaiQ==" } } }, @@ -4992,9 +22813,9 @@ }, "dependencies": { "domelementtype": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.0.1.tgz", - "integrity": "sha512-5HOHUDsYZWV8FGWN0Njbr/Rn7f/eWSQi1v7+HsUVwXgn8nWWlL64zKDkS0n8ZmQ3mlWOMuXOnR+7Nx/5tMO5AQ==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.1.0.tgz", + "integrity": "sha512-LsTgx/L5VpD+Q8lmsXSHW2WpA+eBlZ9HPf3erD1IoPF00/3JKHZ3BknUVA2QGDNu69ZNmyFmCWBSO45XjYKC5w==" } } }, @@ -5094,14 +22915,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -5120,9 +22933,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "electron-to-chromium": { - "version": "1.3.555", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.555.tgz", - "integrity": "sha512-/55x3nF2feXFZ5tdGUOr00TxnUjUgdxhrn+eCJ1FAcoAt+cKQTjQkUC5XF4frMWE1R5sjHk+JueuBalimfe5Pg==" + "version": "1.3.615", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.615.tgz", + "integrity": "sha512-fNYTQXoUhNc6RmHDlGN4dgcLURSBIqQCN7ls6MuQ741+NJyLNRz8DxAC+pZpOKfRs6cfY0lv2kWdy8Oxf9j4+A==" }, "elliptic": { "version": "6.5.3", @@ -5205,21 +23018,13 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, "entities": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz", - "integrity": "sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", + "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==" }, "errno": { "version": "0.1.7", @@ -5238,21 +23043,21 @@ } }, "es-abstract": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", - "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1", - "is-callable": "^1.1.5", - "is-regex": "^1.0.5", - "object-inspect": "^1.7.0", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", - "string.prototype.trimleft": "^2.1.1", - "string.prototype.trimright": "^2.1.1" + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" } }, "es-to-primitive": { @@ -5295,9 +23100,9 @@ } }, "escalade": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.0.2.tgz", - "integrity": "sha512-gPYAU37hYCUhW5euPeR+Y74F7BL+IBsV93j5cvGriSaD1aG6MGsqsV1yamRdrWrb2j3aiZvb0X+UBOWpx3JWtQ==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, "escape-html": { "version": "1.0.3", @@ -5426,9 +23231,9 @@ "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, "import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", "requires": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -5770,10 +23575,11 @@ } }, "resolve": { - "version": "1.17.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", - "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.19.0.tgz", + "integrity": "sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==", "requires": { + "is-core-module": "^2.1.0", "path-parse": "^1.0.6" } } @@ -5785,11 +23591,11 @@ "integrity": "sha512-iXTCFcOmlWvw4+TOE8CLWj6yX1GwzT0Y6cUfHHZqWnSk144VmVIRcVGtUAzrLES7C798lmvnt02C7rxaOX1HNA==" }, "eslint-scope": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", - "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", "requires": { - "esrecurse": "^4.1.0", + "esrecurse": "^4.3.0", "estraverse": "^4.1.1" } }, @@ -5837,11 +23643,18 @@ } }, "esrecurse": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "requires": { - "estraverse": "^4.1.0" + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==" + } } }, "estraverse": { @@ -5968,24 +23781,6 @@ "jest-regex-util": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -6006,11 +23801,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" } } }, @@ -6420,14 +24210,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -6575,14 +24357,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -6633,14 +24407,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -6675,6 +24441,16 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-intrinsic": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", + "integrity": "sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, "get-own-enumerable-property-symbols": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz", @@ -7001,14 +24777,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -7077,17 +24845,6 @@ "pretty-error": "^2.1.1", "tapable": "^1.1.3", "util.promisify": "1.0.0" - }, - "dependencies": { - "util.promisify": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", - "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", - "requires": { - "define-properties": "^1.1.2", - "object.getownpropertydescriptors": "^2.0.3" - } - } } }, "htmlparser2": { @@ -7215,9 +24972,9 @@ "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==" }, "immer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/immer/-/immer-6.0.2.tgz", - "integrity": "sha512-56CMvUMZl4kkWJFFUe1TjBgGbyb9ibzpLyHD+RSKSVdytuDXgT/HXO1S+GJVywMVl5neGTdAogoR15eRVEd10Q==" + "version": "6.0.9", + "resolved": "https://registry.npmjs.org/immer/-/immer-6.0.9.tgz", + "integrity": "sha512-SyCYnAuiRf67Lvk0VkwFvwtDoEiCMjeamnHvRfnVDyc7re1/rQrNxuL+jJ7lA3WvdC4uznrvbmm+clJ9+XXatg==" }, "import-cwd": { "version": "2.1.0", @@ -7293,40 +25050,67 @@ "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, "inquirer": { - "version": "7.3.3", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz", - "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.0.4.tgz", + "integrity": "sha512-Bu5Td5+j11sCkqfqmUTiwv+tWisMtP0L7Q8WrqA2C/BbBhy1YTdFrvjjlrKq8oagA/tLQBski2Gcx/Sqyi2qSQ==", "requires": { "ansi-escapes": "^4.2.1", - "chalk": "^4.1.0", + "chalk": "^2.4.2", "cli-cursor": "^3.1.0", - "cli-width": "^3.0.0", + "cli-width": "^2.0.0", "external-editor": "^3.0.3", "figures": "^3.0.0", - "lodash": "^4.17.19", + "lodash": "^4.17.15", "mute-stream": "0.0.8", - "run-async": "^2.4.0", - "rxjs": "^6.6.0", + "run-async": "^2.2.0", + "rxjs": "^6.5.3", "string-width": "^4.1.0", - "strip-ansi": "^6.0.0", + "strip-ansi": "^5.1.0", "through": "^2.3.6" }, "dependencies": { - "chalk": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", - "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" + "color-convert": "^1.9.0" } }, - "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", "requires": { - "ansi-regex": "^5.0.0" + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" } } } @@ -7410,9 +25194,9 @@ "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" }, "is-callable": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", + "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==" }, "is-ci": { "version": "2.0.0", @@ -7435,6 +25219,14 @@ "rgba-regex": "^1.0.0" } }, + "is-core-module": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.2.0.tgz", + "integrity": "sha512-XRAfAdyyY5F5cOXn7hYQDqh2Xmii+DEfIcQGxK/uNwMHhIkPWO0g8msXcbzLe+MpGoR951MlqM/2iIlU4vKDdQ==", + "requires": { + "has": "^1.0.3" + } + }, "is-data-descriptor": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", @@ -7561,11 +25353,11 @@ } }, "is-regex": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", - "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", + "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", "requires": { - "has": "^1.0.3" + "has-symbols": "^1.0.1" } }, "is-regexp": { @@ -7724,24 +25516,6 @@ "jest-cli": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -7816,26 +25590,6 @@ "@jest/types": "^24.9.0", "execa": "^1.0.0", "throat": "^4.0.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "jest-config": { @@ -7862,24 +25616,6 @@ "realpath-native": "^1.1.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -7921,11 +25657,6 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -7948,14 +25679,21 @@ } }, "jest-diff": { - "version": "25.2.3", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-25.2.3.tgz", - "integrity": "sha512-VtZ6LAQtaQpFsmEzps15dQc5ELbJxy4L2DOSo2Ev411TUEtnJPkAMD7JneVypeMJQ1y3hgxN9Ao13n15FAnavg==", + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-25.5.0.tgz", + "integrity": "sha512-z1kygetuPiREYdNIumRpAHY6RXiGmp70YHptjdaxTWGmA085W3iCnXNx0DhflK3vwrKmrRWyY1wUpkPMVxMK7A==", "requires": { "chalk": "^3.0.0", - "diff-sequences": "^25.2.1", - "jest-get-type": "^25.2.1", - "pretty-format": "^25.2.3" + "diff-sequences": "^25.2.6", + "jest-get-type": "^25.2.6", + "pretty-format": "^25.5.0" + }, + "dependencies": { + "jest-get-type": { + "version": "25.2.6", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-25.2.6.tgz", + "integrity": "sha512-DxjtyzOHjObRM+sM1knti6or+eOgcGU4xVSb2HNP1TqO4ahsT+rqZg+nyqHWJSvWgKC5cG3QjGFBqxLghiF/Ig==" + } } }, "jest-docblock": { @@ -7978,24 +25716,6 @@ "pretty-format": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -8037,11 +25757,6 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -8074,26 +25789,6 @@ "jest-mock": "^24.9.0", "jest-util": "^24.9.0", "jsdom": "^11.5.1" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "jest-environment-jsdom-fourteen": { @@ -8109,28 +25804,10 @@ "jsdom": "^14.1.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" }, "jsdom": { "version": "14.1.0", @@ -8200,32 +25877,12 @@ "@jest/types": "^24.9.0", "jest-mock": "^24.9.0", "jest-util": "^24.9.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "jest-get-type": { - "version": "25.2.1", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-25.2.1.tgz", - "integrity": "sha512-EYjTiqcDTCRJDcSNKbLTwn/LcDPEE7ITk8yRMNAOjEsN6yp+Uu+V1gx4djwnuj/DvWg0YGmqaBqPVGsPxlvE7w==" + "version": "24.9.0", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", + "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" }, "jest-haste-map": { "version": "24.9.0", @@ -8246,24 +25903,6 @@ "walker": "^1.0.7" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "fsevents": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", @@ -8299,24 +25938,6 @@ "throat": "^4.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -8388,24 +26009,6 @@ "pretty-format": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -8432,11 +26035,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -8461,24 +26059,6 @@ "pretty-format": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -8536,11 +26116,6 @@ "pretty-format": "^24.9.0" } }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -8577,24 +26152,6 @@ "stack-utils": "^1.0.1" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -8647,26 +26204,6 @@ "integrity": "sha512-3BEYN5WbSq9wd+SyLDES7AHnjH9A/ROBwmz7l2y+ol+NtSFO8DYiEBzoO1CeFc9a8DYy10EO4dDFVv/wN3zl1w==", "requires": { "@jest/types": "^24.9.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "jest-pnp-resolver": { @@ -8691,24 +26228,6 @@ "realpath-native": "^1.1.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -8763,26 +26282,6 @@ "@jest/types": "^24.9.0", "jest-regex-util": "^24.3.0", "jest-snapshot": "^24.9.0" - }, - "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - } } }, "jest-runner": { @@ -8811,24 +26310,6 @@ "throat": "^4.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -8905,24 +26386,6 @@ "yargs": "^13.3.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -8994,24 +26457,6 @@ "semver": "^6.2.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -9069,11 +26514,6 @@ "pretty-format": "^24.9.0" } }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -9114,24 +26554,6 @@ "source-map": "^0.6.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -9201,24 +26623,6 @@ "pretty-format": "^24.9.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-regex": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", @@ -9260,11 +26664,6 @@ "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, - "jest-get-type": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-24.9.0.tgz", - "integrity": "sha512-lUseMzAley4LhIcpSP9Jf+fTrQ4a1yHQwLNeeVa2cEmbCGeoZAtYPOIv8JaxLD/sUpKxetKGP+gsHl8f8TSj8Q==" - }, "pretty-format": { "version": "24.9.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-24.9.0.tgz", @@ -9374,24 +26773,6 @@ "string-length": "^2.0.0" }, "dependencies": { - "@jest/types": { - "version": "24.9.0", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-24.9.0.tgz", - "integrity": "sha512-XKK7ze1apu5JWQ5eZjHITP66AX+QsLlbaJRBGYr8pNzwcAE2JVkwnf0yqjHTsDRcjR0mujy/NmZMXw5kl+kGBw==", - "requires": { - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^1.1.1", - "@types/yargs": "^13.0.0" - } - }, - "@types/yargs": { - "version": "13.0.10", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.10.tgz", - "integrity": "sha512-MU10TSgzNABgdzKvQVW1nuuT+sgBMWeXNc3XOs5YXV5SDAK+PPja2eUuBNB9iqElu03xyEDqlnGw0jgl4nbqGQ==", - "requires": { - "@types/yargs-parser": "*" - } - }, "ansi-escapes": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", @@ -9618,9 +26999,9 @@ }, "dependencies": { "csstype": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.2.tgz", - "integrity": "sha512-ofovWglpqoqbfLNOTBNZLSbMuGrblAf1efvvArGKOZMBrIoJeu5UsAipQolkijtyQx5MtAzT/J9IHj/CEY1mJw==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.5.tgz", + "integrity": "sha512-uVDi8LpBUKQj6sdxNaTetL6FpeCqTjOvAQuQUa/qAqq8oOd4ivkbhgnqayl0dnPal8Tb/yB1tF+gOvCBiicaiQ==" } } }, @@ -10027,14 +27408,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -10390,9 +27763,9 @@ } }, "node-forge": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.0.tgz", - "integrity": "sha512-7ASaDa3pD+lJ3WvXFsxekJQelBKRpne+GOVbLbtHYdd7pFspyeuJHnWfLplGf3SwKGbfs/aYl5V/JCIaHVUKKQ==" + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" }, "node-int64": { "version": "0.4.0", @@ -10429,6 +27802,11 @@ "vm-browserify": "^1.0.1" }, "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -10451,16 +27829,6 @@ "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" - }, - "dependencies": { - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - } } }, "util": { @@ -10469,13 +27837,6 @@ "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", "requires": { "inherits": "2.0.3" - }, - "dependencies": { - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - } } } } @@ -10505,9 +27866,9 @@ } }, "node-releases": { - "version": "1.1.60", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz", - "integrity": "sha512-gsO4vjEdQaTusZAEebUWp2a5d7dF5DYoIpDG7WySnk7BuZDW+GPpHXoXXuYawRBr/9t5q54tirPz79kFIWg4dA==" + "version": "1.1.67", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.67.tgz", + "integrity": "sha512-V5QF9noGFl3EymEwUYzO+3NTDpGfQB4ve6Qfnzf3UNydMhjQRVPR1DZTuvWiLzaFJYw2fmDwAfnRNEVb64hSIg==" }, "normalize-package-data": { "version": "2.5.0", @@ -10613,9 +27974,9 @@ "integrity": "sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==" }, "object-inspect": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==" + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.9.0.tgz", + "integrity": "sha512-i3Bp9iTqwhaLZBxGkRfo5ZbE07BQRT7MGu8+nNgwW9ItGp1TzCTw2DLEoWwjClxBjOFI/hWljTAmYGCEwmtnOw==" }, "object-is": { "version": "1.1.2", @@ -10631,11 +27992,6 @@ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, - "object-path": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/object-path/-/object-path-0.11.4.tgz", - "integrity": "sha1-NwrnUvvzfePqcKhhwju6iRVpGUk=" - }, "object-visit": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", @@ -10645,14 +28001,14 @@ } }, "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" } }, "object.entries": { @@ -10893,14 +28249,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -11117,9 +28465,9 @@ }, "dependencies": { "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "requires": { "ms": "^2.1.1" } @@ -11132,9 +28480,9 @@ "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=" }, "postcss": { - "version": "7.0.32", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.32.tgz", - "integrity": "sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw==", + "version": "7.0.35", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.35.tgz", + "integrity": "sha512-3QT8bBJeX/S5zKTTjTCIjRF3If4avAT6kqxcASlTWEtAFCb9NH0OUxNDfgZSWdP5fJnBYCMEWkIFfWeugjzYMg==", "requires": { "chalk": "^2.4.2", "source-map": "^0.6.1", @@ -12050,13 +29398,14 @@ } }, "postcss-selector-parser": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.2.tgz", - "integrity": "sha512-36P2QR59jDTOAiIkqEprfJDsoNrvwFei3eCqKd1Y0tUsBimsq39BLp7RD+JWny3WgB1zGhJX8XVePwm9k4wdBg==", + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", "requires": { "cssesc": "^3.0.0", "indexes-of": "^1.0.1", - "uniq": "^1.0.1" + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" } }, "postcss-svgo": { @@ -12133,14 +29482,35 @@ } }, "pretty-format": { - "version": "25.2.3", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-25.2.3.tgz", - "integrity": "sha512-IP4+5UOAVGoyqC/DiomOeHBUKN6q00gfyT2qpAsRH64tgOKB2yF7FHJXC18OCiU0/YFierACup/zdCOWw0F/0w==", + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-25.5.0.tgz", + "integrity": "sha512-kbo/kq2LQ/A/is0PQwsEHM7Ca6//bGPPvU6UnsdDRSKTWxT/ru/xb88v4BJf6a69H+uTytOEsTusT9ksd/1iWQ==", "requires": { - "@jest/types": "^25.2.3", + "@jest/types": "^25.5.0", "ansi-regex": "^5.0.0", "ansi-styles": "^4.0.0", "react-is": "^16.12.0" + }, + "dependencies": { + "@jest/types": { + "version": "25.5.0", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-25.5.0.tgz", + "integrity": "sha512-OXD0RgQ86Tu3MazKo8bnrkDRaDXXMGUqd+kTtLtK1Zb7CRzQcaSRPPPV37SvYTdevXEBVxe0HXylEjs8ibkmCw==", + "requires": { + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^1.1.1", + "@types/yargs": "^15.0.0", + "chalk": "^3.0.0" + } + }, + "@types/yargs": { + "version": "15.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.11.tgz", + "integrity": "sha512-jfcNBxHFYJ4nPIacsi3woz1+kvUO6s1CyeEhtnDHBjHUMNj5UlW2GynmnSgiJJEdNg9yW5C8lfoNRZrHGv5EqA==", + "requires": { + "@types/yargs-parser": "*" + } + } } }, "process": { @@ -12408,11 +29778,6 @@ "@babel/highlight": "^7.8.3" } }, - "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==" - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -12449,11 +29814,6 @@ } } }, - "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==" - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -12506,36 +29866,6 @@ "resolved": "https://registry.npmjs.org/immer/-/immer-1.10.0.tgz", "integrity": "sha512-O3sR1/opvCDGLEVcvrGTMtLac8GJ5IwZC4puPrLuRj3l7ICKvkmA0vGuU9OW8mV9WIBRnaxp5GJh9IEAaNOoYg==" }, - "inquirer": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.0.4.tgz", - "integrity": "sha512-Bu5Td5+j11sCkqfqmUTiwv+tWisMtP0L7Q8WrqA2C/BbBhy1YTdFrvjjlrKq8oagA/tLQBski2Gcx/Sqyi2qSQ==", - "requires": { - "ansi-escapes": "^4.2.1", - "chalk": "^2.4.2", - "cli-cursor": "^3.1.0", - "cli-width": "^2.0.0", - "external-editor": "^3.0.3", - "figures": "^3.0.0", - "lodash": "^4.17.15", - "mute-stream": "0.0.8", - "run-async": "^2.2.0", - "rxjs": "^6.5.3", - "string-width": "^4.1.0", - "strip-ansi": "^5.1.0", - "through": "^2.3.6" - }, - "dependencies": { - "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", - "requires": { - "ansi-regex": "^4.1.0" - } - } - } - }, "json5": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", @@ -12691,9 +30021,9 @@ } }, "react-scripts": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/react-scripts/-/react-scripts-3.4.3.tgz", - "integrity": "sha512-oSnoWmii/iKdeQiwaO6map1lUaZLmG0xIUyb/HwCVFLT7gNbj8JZ9RmpvMCZ4fB98ZUMRfNmp/ft8uy/xD1RLA==", + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/react-scripts/-/react-scripts-3.4.4.tgz", + "integrity": "sha512-7J7GZyF/QvZkKAZLneiOIhHozvOMHey7hO9cdO9u68jjhGZlI8hDdOm6UyuHofn6Ajc9Uji5I6Psm/nKNuWdyw==", "requires": { "@babel/core": "7.9.0", "@svgr/webpack": "4.3.3", @@ -12737,7 +30067,7 @@ "react-app-polyfill": "^1.0.6", "react-dev-utils": "^10.2.1", "resolve": "1.15.0", - "resolve-url-loader": "3.1.1", + "resolve-url-loader": "3.1.2", "sass-loader": "8.0.2", "semver": "6.3.0", "style-loader": "0.23.1", @@ -12791,9 +30121,9 @@ } }, "readdirp": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", - "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", "requires": { "picomatch": "^2.2.1" } @@ -12829,9 +30159,9 @@ "integrity": "sha512-km6dclyFnmcvxhAcrQV2AkZmPQjzPDjgVlQtR0EQjxZPyJ0BnMf3in1ryuR8A2qU0HldVRfxYXbFSKlI3N7Slw==" }, "regenerate": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.1.tgz", - "integrity": "sha512-j2+C8+NtXQgEKWk49MMP5P/u2GhnahTtVkRIHr5R5lVRlbKvmQ+oS+A5aLKWp2ma5VkT8sh6v+v4hbH0YHR66A==" + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==" }, "regenerate-unicode-properties": { "version": "8.2.0", @@ -12842,9 +30172,9 @@ } }, "regenerator-runtime": { - "version": "0.13.5", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", - "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==" + "version": "0.13.7", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" }, "regenerator-transform": { "version": "0.14.5", @@ -12864,9 +30194,9 @@ } }, "regex-parser": { - "version": "2.2.10", - "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.10.tgz", - "integrity": "sha512-8t6074A68gHfU8Neftl0Le6KTDwfGAj7IyjPIMSfikI2wJUTHDMaIq42bUsfVnj8mhx0R+45rdUXHGpN164avA==" + "version": "2.2.11", + "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.11.tgz", + "integrity": "sha512-jbD/FT0+9MBU2XAZluI7w2OBs1RBi6p9M83nkoZayQXXU9e8Robt69FcZc7wU4eJD/YFTjn1JdCk3rbMJajz8Q==" }, "regexp.prototype.flags": { "version": "1.3.0", @@ -12883,9 +30213,9 @@ "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==" }, "regexpu-core": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.7.0.tgz", - "integrity": "sha512-TQ4KXRnIn6tz6tjnrXEkD/sshygKH/j5KzK86X8MkeHyZ8qst/LZ89j3X4/8HEIfHANTFIP/AbXakeRhWIl5YQ==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.7.1.tgz", + "integrity": "sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ==", "requires": { "regenerate": "^1.4.0", "regenerate-unicode-properties": "^8.2.0", @@ -13084,11 +30414,11 @@ "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" }, "resolve-url-loader": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-3.1.1.tgz", - "integrity": "sha512-K1N5xUjj7v0l2j/3Sgs5b8CjrrgtC70SmdCuZiJ8tSyb5J+uk3FoeZ4b7yTnH6j7ngI+Bc5bldHJIa8hYdu2gQ==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-3.1.2.tgz", + "integrity": "sha512-QEb4A76c8Mi7I3xNKXlRKQSlLBwjUV/ULFMP+G7n3/7tJZ8MG5wsZ3ucxP1Jz8Vevn6fnJsxDx9cIls+utGzPQ==", "requires": { - "adjust-sourcemap-loader": "2.0.0", + "adjust-sourcemap-loader": "3.0.0", "camelcase": "5.3.1", "compose-function": "3.0.3", "convert-source-map": "1.7.0", @@ -13401,11 +30731,11 @@ "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=" }, "selfsigned": { - "version": "1.10.7", - "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.7.tgz", - "integrity": "sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA==", + "version": "1.10.8", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz", + "integrity": "sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w==", "requires": { - "node-forge": "0.9.0" + "node-forge": "^0.10.0" } }, "semver": { @@ -13633,41 +30963,23 @@ }, "dependencies": { "es-abstract": { - "version": "1.18.0-next.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.0.tgz", - "integrity": "sha512-elZXTZXKn51hUBdJjSZGYRujuzilgXo8vSPQzjGYXLvSlGiCo8VO8ZGV3kjo9a0WNJJ57hENagwbtlRuHuzkcQ==", + "version": "1.18.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", + "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", "has": "^1.0.3", "has-symbols": "^1.0.1", - "is-callable": "^1.2.0", + "is-callable": "^1.2.2", "is-negative-zero": "^2.0.0", "is-regex": "^1.1.1", "object-inspect": "^1.8.0", "object-keys": "^1.1.1", - "object.assign": "^4.1.0", + "object.assign": "^4.1.1", "string.prototype.trimend": "^1.0.1", "string.prototype.trimstart": "^1.0.1" } - }, - "is-callable": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", - "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==" - }, - "is-regex": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", - "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", - "requires": { - "has-symbols": "^1.0.1" - } - }, - "object-inspect": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", - "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==" } } }, @@ -13868,9 +31180,9 @@ }, "dependencies": { "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", "requires": { "ms": "^2.1.1" } @@ -14093,14 +31405,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -14143,14 +31447,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -14164,6 +31460,14 @@ "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "string-length": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/string-length/-/string-length-2.0.0.tgz", @@ -14230,24 +31534,6 @@ "es-abstract": "^1.17.5" } }, - "string.prototype.trimleft": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.1.tgz", - "integrity": "sha512-iu2AGd3PuP5Rp7x2kEZCrB2Nf41ehzh+goo8TV7z8/XDBbsvc6HQIlUl9RjkZ4oyrW1XM5UwlGl1oVEaDjg6Ag==", - "requires": { - "define-properties": "^1.1.3", - "function-bind": "^1.1.1" - } - }, - "string.prototype.trimright": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.1.tgz", - "integrity": "sha512-qFvWL3/+QIgZXVmJBfpHmxLB7xsUXz6HsUmP8+5dRaC3Q7oKUv9Vo6aMCRZC1smrtyECFsIT30PqBJ1gTjAs+g==", - "requires": { - "define-properties": "^1.1.3", - "function-bind": "^1.1.1" - } - }, "string.prototype.trimstart": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", @@ -14257,21 +31543,6 @@ "es-abstract": "^1.17.5" } }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" - } - } - }, "stringify-object": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", @@ -14372,9 +31643,9 @@ } }, "supports-color": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", - "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "requires": { "has-flag": "^4.0.0" } @@ -14656,14 +31927,6 @@ "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -15047,14 +32310,12 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "util.promisify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.1.tgz", - "integrity": "sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz", + "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==", "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.2", - "has-symbols": "^1.0.1", - "object.getownpropertydescriptors": "^2.1.0" + "define-properties": "^1.1.2", + "object.getownpropertydescriptors": "^2.0.3" } }, "utila": { @@ -15265,15 +32526,6 @@ "micromatch": "^3.1.10", "readable-stream": "^2.0.2" } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "optional": true, - "requires": { - "safe-buffer": "~5.1.0" - } } } }, @@ -15321,9 +32573,9 @@ }, "dependencies": { "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==" + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" }, "cacache": { "version": "12.0.4", @@ -15569,14 +32821,6 @@ "ajv-keywords": "^3.1.0" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/dashboard/client/src/pages/dashboard/node-info/features/ObjectStoreMemory.tsx b/dashboard/client/src/pages/dashboard/node-info/features/ObjectStoreMemory.tsx index 0201040ff..95f4421b1 100644 --- a/dashboard/client/src/pages/dashboard/node-info/features/ObjectStoreMemory.tsx +++ b/dashboard/client/src/pages/dashboard/node-info/features/ObjectStoreMemory.tsx @@ -18,20 +18,22 @@ export const ClusterObjectStoreMemory: ClusterFeatureRenderFn = ({ nodes }) => { nodes.map((n) => n.raylet.objectStoreAvailableMemory), ); const totalUsed = sum(nodes.map((n) => n.raylet.objectStoreUsedMemory)); + const total = totalUsed + totalAvailable; return (
); }; export const NodeObjectStoreMemory: NodeFeatureRenderFn = ({ node }) => { - const total = node.raylet.objectStoreAvailableMemory; + const totalAvailable = node.raylet.objectStoreAvailableMemory; const used = node.raylet.objectStoreUsedMemory; - if (used === undefined || total === undefined || total === 0) { + const total = totalAvailable + used; + if (used === undefined || totalAvailable === undefined || total === 0) { return ( N/A diff --git a/dashboard/dashboard.py b/dashboard/dashboard.py index 5fecebdb5..80765b647 100644 --- a/dashboard/dashboard.py +++ b/dashboard/dashboard.py @@ -207,4 +207,5 @@ if __name__ == "__main__": if isinstance(e, OSError) and e.errno == errno.ENOENT: logger.warning(message) else: + logger.exception(message) raise e diff --git a/dashboard/datacenter.py b/dashboard/datacenter.py index 553ff713b..357bab3c1 100644 --- a/dashboard/datacenter.py +++ b/dashboard/datacenter.py @@ -238,7 +238,7 @@ class DataOrganizer: pid = core_worker_stats.get("pid") node_physical_stats = DataSource.node_physical_stats.get(node_id, {}) actor_process_stats = None - actor_process_gpu_stats = None + actor_process_gpu_stats = [] if pid: for process_stats in node_physical_stats.get("workers", []): if process_stats["pid"] == pid: @@ -248,14 +248,11 @@ class DataOrganizer: for gpu_stats in node_physical_stats.get("gpus", []): for process in gpu_stats.get("processes", []): if process["pid"] == pid: - actor_process_gpu_stats = gpu_stats + actor_process_gpu_stats.append(gpu_stats) break - if actor_process_gpu_stats is not None: - break actor["gpus"] = actor_process_gpu_stats actor["processStats"] = actor_process_stats - return actor @classmethod diff --git a/dashboard/modules/logical_view/tests/test_logical_view_head.py b/dashboard/modules/logical_view/tests/test_logical_view_head.py index 2db29b76d..2144918a4 100644 --- a/dashboard/modules/logical_view/tests/test_logical_view_head.py +++ b/dashboard/modules/logical_view/tests/test_logical_view_head.py @@ -54,6 +54,13 @@ def test_actor_groups(ray_start_with_dashboard): assert summary["stateToCount"]["ALIVE"] == 2 entries = actor_groups["Foo"]["entries"] + foo_entry = entries[0] + assert type(foo_entry["gpus"]) is list + assert "timestamp" in foo_entry + assert "actorConstructor" in foo_entry + assert "actorClass" in foo_entry + assert "actorId" in foo_entry + assert "ipAddress" in foo_entry assert len(entries) == 2 assert "InfeasibleActor" in actor_groups diff --git a/dashboard/modules/stats_collector/stats_collector_head.py b/dashboard/modules/stats_collector/stats_collector_head.py index c819f988c..1224b6d62 100644 --- a/dashboard/modules/stats_collector/stats_collector_head.py +++ b/dashboard/modules/stats_collector/stats_collector_head.py @@ -147,20 +147,22 @@ class StatsCollector(dashboard_utils.DashboardHeadModule): @routes.get("/node_logs") async def get_logs(self, req) -> aiohttp.web.Response: ip = req.query["ip"] - pid = req.query.get("pid") - node_logs = DataSource.ip_and_pid_to_logs[ip] - payload = node_logs.get(pid, []) if pid else node_logs + pid = str(req.query.get("pid", "")) + node_logs = DataSource.ip_and_pid_to_logs.get(ip, {}) + if pid: + node_logs = {str(pid): node_logs.get(pid, [])} return dashboard_utils.rest_response( - success=True, message="Fetched logs.", logs=payload) + success=True, message="Fetched logs.", logs=node_logs) @routes.get("/node_errors") async def get_errors(self, req) -> aiohttp.web.Response: ip = req.query["ip"] - pid = req.query.get("pid") - node_errors = DataSource.ip_and_pid_to_errors[ip] - filtered_errs = node_errors.get(pid, []) if pid else node_errors + pid = str(req.query.get("pid", "")) + node_errors = DataSource.ip_and_pid_to_errors.get(ip, {}) + if pid: + node_errors = {str(pid): node_errors.get(pid, [])} return dashboard_utils.rest_response( - success=True, message="Fetched errors.", errors=filtered_errs) + success=True, message="Fetched errors.", errors=node_errors) async def _update_actors(self): # Subscribe actor channel. diff --git a/dashboard/modules/stats_collector/tests/test_stats_collector.py b/dashboard/modules/stats_collector/tests/test_stats_collector.py index 5771e7e04..f4246770a 100644 --- a/dashboard/modules/stats_collector/tests/test_stats_collector.py +++ b/dashboard/modules/stats_collector/tests/test_stats_collector.py @@ -4,15 +4,16 @@ import logging import requests import time import traceback - +import random import pytest import ray +import threading +from datetime import datetime, timedelta +from ray.cluster_utils import Cluster from ray.new_dashboard.tests.conftest import * # noqa -from ray.test_utils import ( - format_web_url, - wait_until_server_available, - wait_for_condition, -) +from ray.test_utils import (format_web_url, wait_until_server_available, + wait_for_condition, + wait_until_succeeded_without_exception) logger = logging.getLogger(__name__) @@ -184,7 +185,7 @@ def test_get_all_node_details(disable_aiohttp_cache, ray_start_with_dashboard): }], indirect=True) def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache, ray_start_cluster_head): - cluster = ray_start_cluster_head + cluster: Cluster = ray_start_cluster_head assert (wait_until_server_available(cluster.webui_url) is True) webui_url = cluster.webui_url webui_url = format_web_url(webui_url) @@ -216,7 +217,164 @@ def test_multi_nodes_info(enable_test_module, disable_aiohttp_cache, logger.info(ex) return False - wait_for_condition(_check_nodes, timeout=10) + wait_for_condition(_check_nodes, timeout=15) + + +@pytest.mark.parametrize( + "ray_start_cluster_head", [{ + "include_dashboard": True + }], indirect=True) +def test_multi_node_churn(enable_test_module, disable_aiohttp_cache, + ray_start_cluster_head): + cluster: Cluster = ray_start_cluster_head + assert (wait_until_server_available(cluster.webui_url) is True) + webui_url = format_web_url(cluster.webui_url) + + def cluster_chaos_monkey(): + worker_nodes = [] + while True: + time.sleep(5) + if len(worker_nodes) < 2: + worker_nodes.append(cluster.add_node()) + continue + should_add_node = random.randint(0, 1) + if should_add_node: + worker_nodes.append(cluster.add_node()) + else: + node_index = random.randrange(0, len(worker_nodes)) + node_to_remove = worker_nodes.pop(node_index) + cluster.remove_node(node_to_remove) + + def get_index(): + resp = requests.get(webui_url) + resp.raise_for_status() + + def get_nodes(): + resp = requests.get(webui_url + "/nodes?view=summary") + resp.raise_for_status() + summary = resp.json() + assert summary["result"] is True, summary["msg"] + assert summary["data"]["summary"] + + t = threading.Thread(target=cluster_chaos_monkey, daemon=True) + t.start() + + t_st = datetime.now() + duration = timedelta(seconds=60) + while datetime.now() < t_st + duration: + get_index() + time.sleep(2) + + +@pytest.mark.parametrize( + "ray_start_cluster_head", [{ + "include_dashboard": True + }], indirect=True) +def test_logs(enable_test_module, disable_aiohttp_cache, + ray_start_cluster_head): + cluster = ray_start_cluster_head + assert (wait_until_server_available(cluster.webui_url) is True) + webui_url = cluster.webui_url + webui_url = format_web_url(webui_url) + nodes = ray.nodes() + assert len(nodes) == 1 + node_ip = nodes[0]["NodeManagerAddress"] + + @ray.remote + class LoggingActor: + def go(self, n): + i = 0 + while i < n: + print(f"On number {i}") + i += 1 + + def get_pid(self): + return os.getpid() + + la = LoggingActor.remote() + la2 = LoggingActor.remote() + la_pid = str(ray.get(la.get_pid.remote())) + la2_pid = str(ray.get(la2.get_pid.remote())) + ray.get(la.go.remote(4)) + ray.get(la2.go.remote(1)) + + def check_logs(): + node_logs_response = requests.get( + f"{webui_url}/node_logs", params={"ip": node_ip}) + node_logs_response.raise_for_status() + node_logs = node_logs_response.json() + assert node_logs["result"] + assert type(node_logs["data"]["logs"]) is dict + assert all( + pid in node_logs["data"]["logs"] for pid in (la_pid, la2_pid)) + assert len(node_logs["data"]["logs"][la2_pid]) == 1 + + actor_one_logs_response = requests.get( + f"{webui_url}/node_logs", + params={ + "ip": node_ip, + "pid": str(la_pid) + }) + actor_one_logs_response.raise_for_status() + actor_one_logs = actor_one_logs_response.json() + assert actor_one_logs["result"] + assert type(actor_one_logs["data"]["logs"]) is dict + assert len(actor_one_logs["data"]["logs"][la_pid]) == 4 + + wait_until_succeeded_without_exception( + check_logs, (AssertionError), timeout_ms=1000) + + +@pytest.mark.parametrize( + "ray_start_cluster_head", [{ + "include_dashboard": True + }], indirect=True) +def test_errors(enable_test_module, disable_aiohttp_cache, + ray_start_cluster_head): + cluster = ray_start_cluster_head + assert (wait_until_server_available(cluster.webui_url) is True) + webui_url = cluster.webui_url + webui_url = format_web_url(webui_url) + nodes = ray.nodes() + assert len(nodes) == 1 + node_ip = nodes[0]["NodeManagerAddress"] + + @ray.remote + class ErrorActor(): + def go(self): + raise ValueError("This is an error") + + def get_pid(self): + return os.getpid() + + ea = ErrorActor.remote() + ea_pid = ea.get_pid.remote() + ea.go.remote() + + def check_errs(): + node_errs_response = requests.get( + f"{webui_url}/node_logs", params={"ip": node_ip}) + node_errs_response.raise_for_status() + node_errs = node_errs_response.json() + assert node_errs["result"] + assert type(node_errs["data"]["errors"]) is dict + assert ea_pid in node_errs["data"]["errors"] + assert len(node_errs["data"]["errors"][ea_pid]) == 1 + + actor_err_response = requests.get( + f"{webui_url}/node_logs", + params={ + "ip": node_ip, + "pid": str(ea_pid) + }) + actor_err_response.raise_for_status() + actor_errs = actor_err_response.json() + assert actor_errs["result"] + assert type(actor_errs["data"]["errors"]) is dict + assert len(actor_errs["data"]["errors"][ea_pid]) == 4 + + wait_until_succeeded_without_exception( + check_errs, (AssertionError), timeout_ms=1000) if __name__ == "__main__": diff --git a/dashboard/tests/test_dashboard.py b/dashboard/tests/test_dashboard.py index a37fd3012..32836c0ae 100644 --- a/dashboard/tests/test_dashboard.py +++ b/dashboard/tests/test_dashboard.py @@ -80,7 +80,7 @@ def test_basic(ray_start_with_dashboard): 0] dashboard_proc = psutil.Process(dashboard_proc_info.process.pid) assert dashboard_proc.status() in [ - psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING + psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING, psutil.STATUS_DISK_SLEEP ] raylet_proc_info = all_processes[ray_constants.PROCESS_TYPE_RAYLET][0] raylet_proc = psutil.Process(raylet_proc_info.process.pid) @@ -137,6 +137,11 @@ def test_basic(ray_start_with_dashboard): assert agent_proc.pid == agent_pid time.sleep(1) + # The agent should be dead if raylet exits. + raylet_proc.kill() + raylet_proc.wait() + agent_proc.wait(5) + # Check redis keys are set. logger.info("Check redis keys are set.") dashboard_address = client.get(dashboard_consts.REDIS_KEY_DASHBOARD) diff --git a/doc/examples/cython/ray-project/requirements.txt b/doc/examples/cython/ray-project/requirements.txt index 43eabb232..8755fced6 100644 --- a/doc/examples/cython/ray-project/requirements.txt +++ b/doc/examples/cython/ray-project/requirements.txt @@ -1,2 +1,2 @@ -ray[debug] +ray scipy diff --git a/doc/examples/lbfgs/ray-project/requirements.txt b/doc/examples/lbfgs/ray-project/requirements.txt index dc3a91b64..f740f42b6 100644 --- a/doc/examples/lbfgs/ray-project/requirements.txt +++ b/doc/examples/lbfgs/ray-project/requirements.txt @@ -1 +1 @@ -ray[debug,rllib] +ray[rllib] diff --git a/doc/examples/newsreader/ray-project/cluster.yaml b/doc/examples/newsreader/ray-project/cluster.yaml index 6b69fe05a..848b08c41 100644 --- a/doc/examples/newsreader/ray-project/cluster.yaml +++ b/doc/examples/newsreader/ray-project/cluster.yaml @@ -24,7 +24,7 @@ setup_commands: - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc - sudo apt -y update - sudo apt -y install npm - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl # How Ray will authenticate with newly launched nodes. auth: diff --git a/doc/examples/newsreader/ray-project/requirements.txt b/doc/examples/newsreader/ray-project/requirements.txt index 9040cd6cc..b9f0e9c76 100644 --- a/doc/examples/newsreader/ray-project/requirements.txt +++ b/doc/examples/newsreader/ray-project/requirements.txt @@ -1,3 +1,3 @@ -ray[debug] +ray atoma flask diff --git a/doc/examples/placement-group.rst b/doc/examples/placement-group.rst index 3e833dd06..f552e6f6e 100644 --- a/doc/examples/placement-group.rst +++ b/doc/examples/placement-group.rst @@ -28,9 +28,10 @@ Gang Scheduling --------------- **Recommended Strategy**: `STRICT_SPREAD`. -Sometimes, you'd like to schedule multiple tasks/actors in a separate physical machine (node) "at the same time". For example, "write the gang scheduling example". +Sometimes, you'd like to schedule multiple tasks/actors in a separate physical machine (node) "at the same time". For example, you may use a collective communication library like NCCL and form a communication group with a set of actors. In creating this set of actors, you may want to ensure that all actors are separated and placed evenly across available nodes (creating a homogenous cluster). You can use placement groups' `STRICT_SPREAD` strategy to achieve it. `STRICT_SPREAD` ensures that all actors and tasks scheduled with the placement group will be located in a separate node. +Also, since the placement group creation is atomic, you can always guarantee that tasks and actors are scheduled at the same time. Improve Fault tolerance ----------------------- diff --git a/doc/examples/streaming/ray-project/requirements.txt b/doc/examples/streaming/ray-project/requirements.txt index 99bbe32ce..bec8c5bdc 100644 --- a/doc/examples/streaming/ray-project/requirements.txt +++ b/doc/examples/streaming/ray-project/requirements.txt @@ -1,2 +1,2 @@ -ray[debug] +ray wikipedia diff --git a/doc/requirements-doc.txt b/doc/requirements-doc.txt index 6de01b0c9..cb2c358fa 100644 --- a/doc/requirements-doc.txt +++ b/doc/requirements-doc.txt @@ -25,6 +25,7 @@ sphinx-jsonschema sphinx-tabs sphinx-version-warning sphinx-book-theme +starlette tabulate uvicorn werkzeug diff --git a/doc/source/cluster/autoscaling.rst b/doc/source/cluster/autoscaling.rst index 996413e8f..e8d8f235d 100644 --- a/doc/source/cluster/autoscaling.rst +++ b/doc/source/cluster/autoscaling.rst @@ -10,7 +10,7 @@ Basics The Ray Cluster Launcher will automatically enable a load-based autoscaler. The scheduler will look at the task, actor, and placement group resource demands from the cluster, and tries to add the minimum set of nodes that can fulfill these demands. When nodes are idle for more than a timeout, they will be removed, down to the ``min_workers`` limit. The head node is never removed. -To avoid launching too many nodes at once, the number of nodes allowed to be pending is limited by the ``upscaling_speed`` setting. By default it is set to ``1.0``, which means the cluster can grow in size by at most ``100%`` at a time (doubling in size each time). This fraction can be set to as high as needed, e.g., ``99999`` to allow the cluster to quickly grow to its max size. +To avoid launching too many nodes at once, the number of nodes allowed to be pending is limited by the ``upscaling_speed`` setting. By default it is set to ``1.0``, which means the cluster can be growing in size by at most ``100%`` at any time (e.g., if the cluster currently has 20 nodes, at most 20 pending launches are allowed). This fraction can be set to as high as needed, e.g., ``99999`` to allow the cluster to quickly grow to its max size. In more detail, the autoscaler implements the following control loop: @@ -124,7 +124,7 @@ The node config tells the underlying Cloud provider how to launch a node of this node_config: InstanceType: p2.xlarge -The resources field tells the autoscaler what kinds of resources this node provides. This can include custom resources as well (e.g., "Custom2"). This field enables the autoscaler to automatically select the right kind of nodes to launch given the resource demands of the application. The resources specified here will be automatically passed to the ``ray start`` command for the node via an environment variable. For more information, see also the `resource demand scheduler `__: +The resources field tells the autoscaler what kinds of resources this node provides. This can include custom resources as well (e.g., "Custom2"). This field enables the autoscaler to automatically select the right kind of nodes to launch given the resource demands of the application. The resources specified here will be automatically passed to the ``ray start`` command for the node via an environment variable. For more information, see also the `resource demand scheduler `__: .. code-block:: yaml diff --git a/doc/source/cluster/index.rst b/doc/source/cluster/index.rst index 529c4993d..c95eca1cb 100644 --- a/doc/source/cluster/index.rst +++ b/doc/source/cluster/index.rst @@ -21,26 +21,34 @@ Clusters are started with the :ref:`Ray Cluster Launcher You can also create a Ray cluster using a standard cluster manager such as :ref:`Kubernetes `, :ref:`YARN `, or :ref:`SLURM `. -After a cluster is started, you need to connect your program to the Ray cluster. +After a cluster is started, you need to connect your program to the Ray cluster by starting a driver process on the same node as where you ran ``ray start``: .. tabs:: - .. group-tab:: python + .. code-tab:: python - You can connect to this Ray runtime by starting a Python process that calls the following on the same node as where you ran ``ray start``: - - .. code-block:: python - - # This must - import ray - ray.init(address='auto') + # This must + import ray + ray.init(address='auto') .. group-tab:: java - If you want to run Java code, you need to specify the classpath via the ``--code-search-path`` option. See :ref:`code_search_path` for more details. + .. code-block:: java + + import io.ray.api.Ray; + + public class MyRayApp { + + public static void main(String[] args) { + Ray.init(); + ... + } + } .. code-block:: bash - $ ray start ... --code-search-path=/path/to/jars + java -classpath \ + -Dray.address=
\ + and then the rest of your script should be able to leverage Ray as a distributed framework! @@ -74,8 +82,6 @@ The most preferable way to run a Ray cluster is via the :ref:`Ray Cluster Launch This section assumes that you have a list of machines and that the nodes in the cluster can communicate with each other. It also assumes that Ray is installed on each machine. To install Ray, follow the `installation instructions`_. -To configure the Ray cluster to run Java code, you need to add the ``--code-search-path`` option. See :ref:`code_search_path` for more details. - .. _`installation instructions`: http://docs.ray.io/en/master/installation.html Starting Ray on each machine @@ -199,7 +205,7 @@ To run a distributed Ray program, you'll need to execute your program on the sam .. code-block:: bash - java -classpath /path/to/jars/ \ + java -classpath \ -Dray.address=
\ diff --git a/doc/source/cluster/k8s-operator.rst b/doc/source/cluster/k8s-operator.rst new file mode 100644 index 000000000..d17443e55 --- /dev/null +++ b/doc/source/cluster/k8s-operator.rst @@ -0,0 +1,234 @@ +.. _k8s-operator: + +The Ray Kubernetes Operator +================================= + +Ray provides a `Kubernetes Operator`_ for managing autoscaling Ray clusters. +Using the operator provides similar functionality to deploying a Ray cluster using +the :ref:`Ray Cluster Launcher`. However, working with the operator does not require +running Ray locally -- all interactions with your Ray cluster are mediated by Kubernetes. + +The operator makes use of a `Kubernetes Custom Resource`_ called a *RayCluster*. +A RayCluster is specified by a configuration similar to the ``yaml`` files used by the Ray Cluster Launcher. +Internally, the operator uses Ray's autoscaler to manage your Ray cluster. However, the autoscaler runs in a +separate operator pod, rather than on the Ray head node. Applying multiple RayCluster custom resources in the operator's +namespace allows the operator to manage several Ray clusters. + +The rest of this document explains step-by-step how to use the Ray Kubernetes Operator to launch a Ray cluster on your existing Kubernetes cluster. + +.. role:: bash(code) + :language: bash + +.. note:: + The example commands in this document launch six Kubernetes pods, using a total of 6 CPU and 3.5Gi memory. + If you are experimenting using a test Kubernetes environment such as `minikube`_, make sure to provision sufficient resources, e.g. + :bash:`minikube start --cpus=6 --memory=\"4G\"`. + Alternatively, reduce resource usage by editing the ``yaml`` files referenced in this document; for example, reduce ``minWorkers`` + in ``example_cluster.yaml`` and ``example_cluster2.yaml``. + + +Applying the RayCluster Custom Resource Definition +-------------------------------------------------- +First, we need to apply the `Kubernetes Custom Resource Definition`_ (CRD) defining a RayCluster. + +.. note:: + + Creating a Custom Resource Definition requires the appropriate Kubernetes cluster-level privileges. + +.. code-block:: shell + + $ kubectl apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml + + customresourcedefinition.apiextensions.k8s.io/rayclusters.cluster.ray.io created + +Picking a Kubernetes Namespace +------------------------------- +The rest of the Kubernetes resources we will use are `namespaced`_. +You can use an existing namespace for your Ray clusters or create a new one if you have permissions. +For this example, we will create a namespace called ``ray``. + +.. code-block:: shell + + $ kubectl create namespace ray + + namespace/ray created + +Starting the Operator +---------------------- + +To launch the operator in our namespace, we execute the following command. + +.. code-block:: shell + + $ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml + + serviceaccount/ray-operator-serviceaccount created + role.rbac.authorization.k8s.io/ray-operator-role created + rolebinding.rbac.authorization.k8s.io/ray-operator-rolebinding created + pod/ray-operator-pod created + +The output shows that we've launched a Pod named ``ray-operator-pod``. This is the pod that runs the operator process. +The ServiceAccount, Role, and RoleBinding we have created grant the operator pod the `permissions`_ it needs to manage Ray clusters. + +Launching Ray Clusters +---------------------- +Finally, to launch a Ray cluster, we create a RayCluster custom resource. + +.. code-block:: shell + + $ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml + + raycluster.cluster.ray.io/example-cluster created + +The operator detects the RayCluster resource we've created and launches an autoscaling Ray cluster. +Our RayCluster configuration specifies ``minWorkers:2`` in the second entry of ``spec.podTypes``, so we get a head node and two workers upon launch. + +.. note:: + + For more details about RayCluster resources, we recommend take a looking at the annotated example ``example_cluster.yaml`` applied in the last command. + +.. code-block:: shell + + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + example-cluster-ray-head-hbxvv 1/1 Running 0 72s + example-cluster-ray-worker-4hvv6 1/1 Running 0 64s + example-cluster-ray-worker-78kp5 1/1 Running 0 64s + ray-operator-pod 1/1 Running 0 2m33s + +We see four pods: the operator, the Ray head node, and two Ray worker nodes. + +Let's launch another cluster in the same namespace, this one specifiying ``minWorkers:1``. + +.. code-block:: shell + + $ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml + +We confirm that both clusters are running in our namespace. + +.. code-block:: shell + + $ kubectl -n ray get rayclusters + NAME AGE + example-cluster 12m + example-cluster2 114s + + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + example-cluster-ray-head-th4wv 1/1 Running 0 10m + example-cluster-ray-worker-q9pjn 1/1 Running 0 10m + example-cluster-ray-worker-qltnp 1/1 Running 0 10m + example-cluster2-ray-head-kj5mg 1/1 Running 0 10s + example-cluster2-ray-worker-qsgnd 1/1 Running 0 1s + ray-operator-pod 1/1 Running 0 10m + +Now we can :ref:`run Ray programs` on our Ray clusters. + +Monitoring +---------- +Autoscaling logs are written to the operator pod's ``stdout`` and can be accessed with :code:`kubectl logs`. +Each line of output is prefixed by the name of the cluster followed by a colon. +The following command gets the last hundred lines of autoscaling logs for our second cluster. + +.. code-block:: shell + + $ kubectl -n ray logs ray-operator-pod | grep ^example-cluster2: | tail -n 100 + +The output should include monitoring updates that look like this: + +.. code-block:: shell + + example-cluster2:2020-12-12 13:55:36,814 DEBUG autoscaler.py:693 -- Cluster status: 1 nodes + example-cluster2: - MostDelayedHeartbeats: {'172.17.0.4': 0.04093289375305176, '172.17.0.5': 0.04084634780883789} + example-cluster2: - NodeIdleSeconds: Min=36 Mean=38 Max=41 + example-cluster2: - ResourceUsage: 0.0/2.0 CPU, 0.0/1.0 Custom1, 0.0/1.0 is_spot, 0.0 GiB/0.58 GiB memory, 0.0 GiB/0.1 GiB object_store_memory + example-cluster2: - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0 + example-cluster2:Worker node types: + example-cluster2: - worker-nodes: 1 + example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:148 -- Cluster resources: [{'object_store_memory': 1.0, 'node:172.17.0.4': 1.0, 'memory': 5.0, 'CPU': 1.0}, {'object_store_memory': 1.0, 'is_spot': 1.0, 'memory': 6.0, 'node:172.17.0.5': 1.0, 'Custom1': 1.0, 'CPU': 1.0}] + example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:149 -- Node counts: defaultdict(, {'head-node': 1, 'worker-nodes + ': 1}) + example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:159 -- Placement group demands: [] + example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:186 -- Resource demands: [] + example-cluster2:2020-12-12 13:55:36,870 INFO resource_demand_scheduler.py:187 -- Unfulfilled demands: [] + example-cluster2:2020-12-12 13:55:36,891 INFO resource_demand_scheduler.py:209 -- Node requests: {} + example-cluster2:2020-12-12 13:55:36,903 DEBUG autoscaler.py:654 -- example-cluster2-ray-worker-tdxdr is not being updated and passes config check (can_update=True). + example-cluster2:2020-12-12 13:55:36,923 DEBUG autoscaler.py:654 -- example-cluster2-ray-worker-tdxdr is not being updated and passes config check (can_update=True). + + +Updating and Retrying +--------------------- +To update a Ray cluster's configuration, edit the ``yaml`` file of the corresponding RayCluster resource +and apply it again: + +.. code-block:: shell + + $ kubectl -n ray apply -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml + +To force a restart with the same configuration, you can add an `annotation`_ to the RayCluster resource's ``metadata.labels`` field, e.g. + +.. code-block:: yaml + + apiVersion: cluster.ray.io/v1 + kind: RayCluster + metadata: + name: example-cluster + annotations: + try: again + spec: + ... + +Then reapply the RayCluster, as above. + +Currently, editing and reapplying a RayCluster resource will stop and restart Ray processes running on the corresponding +Ray cluster. Similarly, deleting and relaunching the operator pod will stop and restart Ray processes on all Ray clusters in the operator's namespace. +This behavior may be modified in future releases. + + +Cleaning Up +----------- +We shut down a Ray cluster by deleting the associated RayCluster resource. +Either of the next two commands will delete our second cluster ``example-cluster2``. + +.. code-block:: shell + + $ kubectl -n ray delete raycluster example-cluster2 + # OR + $ kubectl -n ray delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml + +The pods associated with ``example-cluster2`` go into ``TERMINATING`` status. In a few moments, we check that these pods are gone: + +.. code-block:: shell + + $ kubectl -n ray get pods + NAME READY STATUS RESTARTS AGE + example-cluster-ray-head-th4wv 1/1 Running 0 57m + example-cluster-ray-worker-q9pjn 1/1 Running 0 56m + example-cluster-ray-worker-qltnp 1/1 Running 0 56m + ray-operator-pod 1/1 Running 0 57m + +Only the operator pod and the first ``example-cluster`` remain. + +To finish clean-up, we delete the cluster ``example-cluster`` and then the operator's resources. + +.. code-block:: shell + + $ kubectl -n ray delete raycluster example-cluster + $ kubectl -n ray delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml + +If you like, you can delete the RayCluster customer resource definition. +(Using the operator again will then require reapplying the CRD.) + +.. code-block:: shell + + $ kubectl delete crd rayclusters.cluster.ray.io + # OR + $ kubectl delete -f ray/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml + +.. _`Kubernetes Operator`: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ +.. _`Kubernetes Custom Resource`: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ +.. _`Kubernetes Custom Resource Definition`: https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/ +.. _`annotation`: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#attaching-metadata-to-objects +.. _`permissions`: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ +.. _`minikube`: https://minikube.sigs.k8s.io/docs/start/ +.. _`namespaced`: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ diff --git a/doc/source/cluster/kubernetes.rst b/doc/source/cluster/kubernetes.rst index f4cb8d517..6793fae0a 100644 --- a/doc/source/cluster/kubernetes.rst +++ b/doc/source/cluster/kubernetes.rst @@ -12,6 +12,9 @@ This document assumes that you have access to a Kubernetes cluster and have first walk you through how to deploy a Ray cluster on your existing Kubernetes cluster, then explore a few different ways to run programs on the Ray cluster. +To learn about deploying an autoscaling Ray cluster using :ref:`Ray's Kubernetes operator`, read +:ref:`here`. + The configuration ``yaml`` files used here are provided in the `Ray repository`_ as examples to get you started. When deploying real applications, you will probably want to build and use your own container images, add more worker nodes to the @@ -38,6 +41,11 @@ flag passed to ``kubectl``. Starting a Ray Cluster ---------------------- +.. toctree:: + :hidden: + + /cluster/k8s-operator.rst + A Ray cluster consists of a single head node and a set of worker nodes (the provided ``ray-cluster.yaml`` file will start 3 worker nodes). In the example Kubernetes configuration, this is implemented as: @@ -142,6 +150,8 @@ and checking that they are restarted by Kubernetes: ray-worker-5c49b7cc57-6m4kp 1/1 Running 0 10s ray-worker-5c49b7cc57-jx2w2 1/1 Running 0 10s +.. _ray-k8s-run: + Running Ray Programs -------------------- diff --git a/doc/source/conf.py b/doc/source/conf.py index 5df6e8be1..c69f73760 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -94,6 +94,9 @@ for mod_name in MOCK_MODULES: sys.modules["tensorflow"].VERSION = "9.9.9" sys.modules["tensorflow.keras.callbacks"] = ChildClassMock() sys.modules["pytorch_lightning"] = ChildClassMock() +sys.modules["xgboost"] = ChildClassMock() +sys.modules["xgboost.core"] = ChildClassMock() +sys.modules["xgboost.callback"] = ChildClassMock() class SimpleClass(object): diff --git a/doc/source/configure.rst b/doc/source/configure.rst index 83a700a9e..5683bf269 100644 --- a/doc/source/configure.rst +++ b/doc/source/configure.rst @@ -60,7 +60,9 @@ If using the command line, connect to the Ray cluster as follow: override this by explicitly setting ``OMP_NUM_THREADS``. ``OMP_NUM_THREADS`` is commonly used in numpy, PyTorch, and Tensorflow to perform multit-threaded linear algebra. In multi-worker setting, we want one thread per worker instead of many threads - per worker to avoid contention. + per worker to avoid contention. Some other libraries may have their own way to configure + parallelism. For example, if you're using OpenCV, you should manually set the number of + threads using cv2.setNumThreads(num_threads) (set to 0 to disable multi-threading). .. _temp-dir-log-files: @@ -243,24 +245,32 @@ Java Applications Code Search Path ~~~~~~~~~~~~~~~~ -If you want to run a Java application in cluster mode, you must first run ``ray start`` to start the Ray cluster. In addition to any ``ray start`` parameters mentioned above, you must add ``--code-search-path`` to tell Ray where to load jars when starting Java workers. Your jar files must be distributed to all nodes of the Ray cluster before running your code, and this parameter must be set on both the head node and non-head nodes. +If you want to run a Java application in a multi-node cluster, you must specify the code search path in your driver. The code search path is to tell Ray where to load jars when starting Java workers. Your jar files must be distributed to the same path(s) on all nodes of the Ray cluster before running your code. .. code-block:: bash - $ ray start ... --code-search-path=/path/to/jars + $ java -classpath \ + -Dray.address=
\ + -Dray.job.code-search-path=/path/to/jars/ \ + -The ``/path/to/jars`` here points to a directory which contains jars. All jars in the directory will be loaded by workers. You can also provide multiple directories for this parameter. +The ``/path/to/jars/`` here points to a directory which contains jars. All jars in the directory will be loaded by workers. You can also provide multiple directories for this parameter. .. code-block:: bash - $ ray start ... --code-search-path=/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2 + $ java -classpath \ + -Dray.address=
\ + -Dray.job.code-search-path=/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2 \ + -Code search path is also used for loading Python code if it's specified. This is required for :ref:`cross_language`. If code search path is specified, you can only run Python remote functions which can be found in the code search path. +You don't need to configure code search path if you run a Java application in a single-node cluster. -You don't need to configure code search path if you run a Java application in single machine mode. +See ``ray.job.code-search-path`` under :ref:`Driver Options ` for more information. .. note:: Currently we don't provide a way to configure Ray when running a Java application in single machine mode. If you need to configure Ray, run ``ray start`` to start the Ray cluster first. +.. _java-driver-options: + Driver Options ~~~~~~~~~~~~~~ @@ -287,4 +297,11 @@ The list of available driver options: - Type: ``Boolean`` - Default: ``false`` +- ``ray.job.code-search-path`` + + - The paths for Java workers to load code from. Currently only directories are supported. You can specify one or more directories split by a ``:``. You don't need to configure code search path if you run a Java application in single machine mode or local mode. Code search path is also used for loading Python code if it's specified. This is required for :ref:`cross_language`. If code search path is specified, you can only run Python remote functions which can be found in the code search path. + - Type: ``String`` + - Default: empty string. + - Example: ``/path/to/jars1:/path/to/jars2:/path/to/pys1:/path/to/pys2`` + .. _`Apache Arrow`: https://arrow.apache.org/ diff --git a/doc/source/cross-language.rst b/doc/source/cross-language.rst index f982233e6..0b742866e 100644 --- a/doc/source/cross-language.rst +++ b/doc/source/cross-language.rst @@ -5,20 +5,46 @@ Cross-language programming This page will show you how to use Ray's cross-language programming feature. -Setup the cluster +Setup the driver ----------------- -We need to set the ``--code-search-path`` option on ``ray start`` command. See :ref:`code_search_path` for more details. +We need to set :ref:`code_search_path` in your driver. -.. code-block:: bash +.. tabs:: - ray start ... --code-search-path=/path/to/code + .. group-tab:: Python + + .. code-block:: python + + ray.init(job_config=ray.job_config.JobConfig(code_search_path="/path/to/code")) + + .. group-tab:: Java + + .. code-block:: bash + + java -classpath \ + -Dray.address=
\ + -Dray.job.code-search-path=/path/to/code/ \ + You may want to include multiple directories to load both Python and Java code for workers, if they are placed in different directories. -.. code-block:: bash +.. tabs:: - ray start ... --code-search-path=/path/to/jars:/path/to/pys + .. group-tab:: Python + + .. code-block:: python + + ray.init(job_config=ray.job_config.JobConfig(code_search_path="/path/to/jars:/path/to/pys")) + + .. group-tab:: Java + + .. code-block:: bash + + java -classpath \ + -Dray.address=
\ + -Dray.job.code-search-path=/path/to/jars:/path/to/pys \ + Python calling Java ------------------- diff --git a/doc/source/development.rst b/doc/source/development.rst index 8719440e7..797b1c770 100644 --- a/doc/source/development.rst +++ b/doc/source/development.rst @@ -23,7 +23,7 @@ RLlib, Tune, Autoscaler, and most Python files do not require you to build and c .. code-block:: shell - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl + pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl 2. Fork and clone the project to your machine. Connect your repository to the upstream (main project) ray repository. diff --git a/doc/source/getting-involved.rst b/doc/source/getting-involved.rst index 20fe2b6bb..2ee0318a2 100644 --- a/doc/source/getting-involved.rst +++ b/doc/source/getting-involved.rst @@ -6,7 +6,10 @@ Getting Involved / Contributing Ray is more than a framework for distributed applications but also an active community of developers, researchers, and folks that love machine learning. -.. tip:: Join our `community slack `_ to discuss Ray! The community is extremely active in helping people succeed in building their ray applications. +.. tip:: Join our `community Slack `_ to + discuss Ray or ask questions on `our forum `_! The + community is extremely active in helping people succeed in building their + Ray applications. You can join (and Star!) us on `on GitHub`_. @@ -141,7 +144,7 @@ You can run the following locally: .. code-block:: shell - ray/scripts/format.sh + ./ci/travis/format.sh An output like the following indicates failure: diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 08b6450b5..40813d86a 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -15,7 +15,7 @@ You can install the latest official version of Ray as follows. Official releases .. code-block:: bash - pip install -U ray # also recommended: ray[debug] + pip install -U ray **Note for Windows Users:** To use Ray on Windows, Visual C++ runtime must be installed (see :ref:`Windows Dependencies ` section). If you run into any issues, please see the :ref:`Windows Support ` section. @@ -55,20 +55,21 @@ instead of the ones above: `Linux Python 3.6`_ `MacOS Python 3.6`_ `Windows Python 3.6`_ =================== =================== ====================== -.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-manylinux2014_x86_64.whl -.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl -.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl -.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl -.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl -.. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl -.. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl -.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-macosx_10_13_intel.whl +.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-manylinux2014_x86_64.whl +.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl +.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl +.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl -.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-win_amd64.whl -.. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-win_amd64.whl -.. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-win_amd64.whl -.. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-win_amd64.whl +.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl +.. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl +.. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-macosx_10_13_intel.whl +.. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-macosx_10_13_intel.whl + +.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-win_amd64.whl +.. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-win_amd64.whl +.. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-win_amd64.whl +.. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-win_amd64.whl Installing from a specific commit @@ -80,11 +81,11 @@ You can install the Ray wheels of any particular commit on ``master`` with the f pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/{COMMIT_HASH}/ray-{RAY_VERSION}-{PYTHON_VERSION}-{PYTHON_VERSION}m-{OS_VERSION}_intel.whl -For example, here are the Ray 1.1.0.dev0 wheels for Python 3.5, MacOS for commit ``a0ba4499ac645c9d3e82e68f3a281e48ad57f873``: +For example, here are the Ray 1.2.0.dev0 wheels for Python 3.5, MacOS for commit ``a0ba4499ac645c9d3e82e68f3a281e48ad57f873``: .. code-block:: bash - pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/a0ba4499ac645c9d3e82e68f3a281e48ad57f873/ray-1.1.0.dev0-cp35-cp35m-macosx_10_13_intel.whl + pip install https://ray-wheels.s3-us-west-2.amazonaws.com/master/a0ba4499ac645c9d3e82e68f3a281e48ad57f873/ray-1.2.0.dev0-cp35-cp35m-macosx_10_13_intel.whl .. _ray-install-java: @@ -139,7 +140,7 @@ The latest Ray Java snapshot can be found in `sonatype repository `_ -- The ``rayproject/ray`` image has ray and all required dependencies. It comes with anaconda and Python 3.7. -- The ``rayproject/autoscaler`` image has the above features as well as many additional libraries. +- The ``rayproject/ray`` `image has ray and all required dependencies. It comes with anaconda and Python 3.7. `_ +- The ``rayproject/ray-ml`` `image has the above features as well as many additional libraries. `_ - The ``rayproject/base-deps`` and ``rayproject/ray-deps`` are for the linux and python dependencies respectively. -These images are tagged by their release number (or commit hash for nightlies) as well as a ``"-gpu"`` if they are GPU compatible. +Image releases are `tagged` using the following format: + + +.. list-table:: + :widths: 25 50 + :header-rows: 1 + + * - Tag + - Description + * - latest + - The most recent Ray release. + * - 1.x.x + - A specific Ray release. + * - nightly + - The most recent Ray build (the most recent commit on Github ``master``) + * - Git SHA + - A specific nightly build (uses a SHA from the Github ``master``). + + +Each tag has `variants` that add or change functionality: + +.. list-table:: + :widths: 16 40 + :header-rows: 1 + + * - Variant + - Description + * - -gpu + - These are based off of an NVIDIA CUDA image. They require the Nvidia Docker Runtime. + * - -cpu + - These are based off of an Ubuntu image. + * - + - Aliases to ``-cpu`` tagged images If you want to tweak some aspect of these images and build them locally, refer to the following script: @@ -308,3 +341,32 @@ that you've cloned the git repository. .. code-block:: bash python -m pytest -v python/ray/tests/test_mini.py + +Troubleshooting +--------------- + +If importing Ray (``python3 -c "import ray"``) in your development clone results +in this error: + +.. code-block:: python + + Traceback (most recent call last): + File "", line 1, in + File ".../ray/python/ray/__init__.py", line 63, in + import ray._raylet # noqa: E402 + File "python/ray/_raylet.pyx", line 98, in init ray._raylet + import ray.memory_monitor as memory_monitor + File ".../ray/python/ray/memory_monitor.py", line 9, in + import psutil # noqa E402 + File ".../ray/python/ray/thirdparty_files/psutil/__init__.py", line 159, in + from . import _psosx as _psplatform + File ".../ray/python/ray/thirdparty_files/psutil/_psosx.py", line 15, in + from . import _psutil_osx as cext + ImportError: cannot import name '_psutil_osx' from partially initialized module 'psutil' (most likely due to a circular import) (.../ray/python/ray/thirdparty_files/psutil/__init__.py) + +Then you should run the following commands: + +.. code-block:: bash + + rm -rf python/ray/thirdparty_files/ + python3 -m pip install setproctitle diff --git a/doc/source/placement-group.rst b/doc/source/placement-group.rst index 3e14fd79f..807cc27e5 100644 --- a/doc/source/placement-group.rst +++ b/doc/source/placement-group.rst @@ -124,7 +124,6 @@ Let's create a placement group. Recall that each bundle is a collection of resou - "CPU" will correspond with `num_cpus` as used in `ray.remote` - "GPU" will correspond with `num_gpus` as used in `ray.remote` - - "MEM" will correspond with `memory` as used in `ray.remote` - Other resources will correspond with `resources` as used in `ray.remote`. Once the placement group reserves resources, original resources are unavailable until the placement group is removed. For example: diff --git a/doc/source/ray-overview/index.rst b/doc/source/ray-overview/index.rst index 25a007d9a..afff2b69c 100644 --- a/doc/source/ray-overview/index.rst +++ b/doc/source/ray-overview/index.rst @@ -196,7 +196,7 @@ RLlib Quick Start .. code-block:: bash pip install tensorflow # or tensorflow-gpu - pip install ray[rllib] # also recommended: ray[debug] + pip install ray[rllib] .. code-block:: python diff --git a/doc/source/ray-overview/involvement.rst b/doc/source/ray-overview/involvement.rst index 31f5f10a7..5f467e536 100644 --- a/doc/source/ray-overview/involvement.rst +++ b/doc/source/ray-overview/involvement.rst @@ -1,7 +1,7 @@ Ray is more than a framework for distributed applications but also an active community of developers, researchers, and folks that love machine learning. Here's a list of tips for getting involved with the Ray community: -- Join our `community slack `_ to discuss Ray! +- Join our `community Slack `_ to discuss Ray! - Star and follow us on `on GitHub`_. - To post questions or feature requests, check out the `Discussion Board`_! - Follow us and spread the word on `Twitter`_! diff --git a/doc/source/rllib.rst b/doc/source/rllib.rst index 79db9095f..984622f27 100644 --- a/doc/source/rllib.rst +++ b/doc/source/rllib.rst @@ -23,7 +23,7 @@ RLlib has extra dependencies on top of ``ray``. First, you'll need to install ei .. code-block:: bash - pip install 'ray[rllib]' # also recommended: ray[debug] + pip install 'ray[rllib]' Then, you can try out training in the following equivalent ways: diff --git a/doc/source/serve/advanced.rst b/doc/source/serve/advanced.rst index 26003559d..01ef54fc9 100644 --- a/doc/source/serve/advanced.rst +++ b/doc/source/serve/advanced.rst @@ -81,6 +81,12 @@ If you *do* want to enable this parallelism in your Serve backend, just set OMP_ client.create_backend("parallel_backend", MyBackend, 12) + +.. note:: + Some other libraries may not respect ``OMP_NUM_THREADS`` and have their own way to configure parallelism. + For example, if you're using OpenCV, you'll need to manually set the number of threads using ``cv2.setNumThreads(num_threads)`` (set to 0 to disable multi-threading). + You can check the configuration using ``cv2.getNumThreads()`` and ``cv2.getNumberOfCPUs()``. + .. _serve-batching: Batching to improve performance @@ -306,12 +312,18 @@ and another named ``ray-tf2`` with Ray Serve and Tensorflow 2. The Ray and python versions must be the same in both environments. To specify an environment for a backend to use, simply pass the environment name in to :mod:`client.create_backend ` -as shown below. Be sure to run the script in an activated conda environment -(not required to be ``ray-tf1`` or ``ray-tf2``). +as shown below. .. literalinclude:: ../../../python/ray/serve/examples/doc/conda_env.py -Alternatively, you may omit the argument ``env`` and call -:mod:`client.create_backend ` -from a script running in the conda environment you want the backend to run in. +.. warning:: + The script must be run in an activated conda environment (not required to be + ``ray-tf1`` or ``ray-tf2``). We hope to remove this restriction in the + future. + +.. note:: + If the argument ``env`` is omitted, backends will be started in the same + conda environment as the caller of + :mod:`client.create_backend ` by + default. diff --git a/doc/source/serve/index.rst b/doc/source/serve/index.rst index 6d4503bb7..b9c0d1497 100644 --- a/doc/source/serve/index.rst +++ b/doc/source/serve/index.rst @@ -37,7 +37,7 @@ Since Serve is built on Ray, it also allows you to scale to many machines, in yo Installation ============ -Ray Serve supports Python versions 3.6 and higher. To install Ray Serve: +Ray Serve supports Python versions 3.6 through 3.8. To install Ray Serve: .. code-block:: bash diff --git a/doc/source/serve/key-concepts.rst b/doc/source/serve/key-concepts.rst index d15a142c8..deada7233 100644 --- a/doc/source/serve/key-concepts.rst +++ b/doc/source/serve/key-concepts.rst @@ -19,7 +19,11 @@ Backends Backends define the implementation of your business logic or models that will handle requests when queries come in to :ref:`serve-endpoint`. In order to support seamless scalability backends can have many replicas, which are individual processes running in the Ray cluster to handle requests. To define a backend, first you must define the "handler" or the business logic you'd like to respond with. -The handler should take as input a `Flask Request object `_ and return any JSON-serializable object as output. +The handler should take as input a `Flask Request object `_. +The handler should return any JSON-serializable object as output. For a more customizable response type, the handler may return a +`Starlette Response object `_. +In the future, Ray Serve will support `Starlette Request objects `_ as input as well. + A backend is defined using :mod:`client.create_backend `, and the implementation can be defined as either a function or a class. Use a function when your response is stateless and a class when you might need to maintain some state (like a model). When using a class, you can specify arguments to be passed to the constructor in :mod:`client.create_backend `, shown below. diff --git a/doc/source/starting-ray.rst b/doc/source/starting-ray.rst index f631d2c78..4a198e27e 100644 --- a/doc/source/starting-ray.rst +++ b/doc/source/starting-ray.rst @@ -125,25 +125,34 @@ Use ``ray start`` from the CLI to start a 1 node ray runtime on a machine. This ... +You can connect to this Ray runtime by starting a driver process on the same node as where you ran ``ray start``: + .. tabs:: - .. group-tab:: python + .. code-tab:: python - You can connect to this Ray runtime by starting a Python process that calls the following on the same node as where you ran ``ray start``: - - .. code-block:: python - - # This must - import ray - ray.init(address='auto') + # This must + import ray + ray.init(address='auto') .. group-tab:: java + .. code-block:: java - If you want to run Java code, you need to specify the classpath via the ``--code-search-path`` option. See :ref:`code_search_path` for more details. + import io.ray.api.Ray; + + public class MyRayApp { + + public static void main(String[] args) { + Ray.init(); + ... + } + } .. code-block:: bash - $ ray start ... --code-search-path=/path/to/jars + java -classpath \ + -Dray.address=
\ + You can connect other nodes to the head node, creating a Ray cluster by also calling ``ray start`` on those nodes. See :ref:`manual-cluster` for more details. Calling ``ray.init(address="auto")`` on any of the cluster machines will connect to the ray cluster. diff --git a/doc/source/tune/api_docs/execution.rst b/doc/source/tune/api_docs/execution.rst index c984d3894..9eebc3c27 100644 --- a/doc/source/tune/api_docs/execution.rst +++ b/doc/source/tune/api_docs/execution.rst @@ -23,14 +23,6 @@ tune.with_parameters .. autofunction:: ray.tune.with_parameters -.. _tune-stop-ref: - -Stopper (tune.Stopper) ----------------------- - -.. autoclass:: ray.tune.Stopper - :members: __call__, stop_all - .. _tune-sync-config: tune.SyncConfig diff --git a/doc/source/tune/api_docs/overview.rst b/doc/source/tune/api_docs/overview.rst index c8cc034a6..cb3f5193c 100644 --- a/doc/source/tune/api_docs/overview.rst +++ b/doc/source/tune/api_docs/overview.rst @@ -21,6 +21,7 @@ on `Github`_. suggestion.rst schedulers.rst sklearn.rst + stoppers.rst logging.rst integration.rst internals.rst diff --git a/doc/source/tune/api_docs/stoppers.rst b/doc/source/tune/api_docs/stoppers.rst new file mode 100644 index 000000000..4d65754be --- /dev/null +++ b/doc/source/tune/api_docs/stoppers.rst @@ -0,0 +1,46 @@ +.. _tune-stoppers: + +Stopping mechanisms (tune.stopper) +================================== + +In addition to Trial Schedulers like :ref:`ASHA `, where a number of +trials are stopped if they perform subpar, Ray Tune also supports custom stopping mechanisms to stop trials early. For instance, stopping mechanisms can specify to stop trials when they reached a plateau and the metric +doesn't change anymore. + +Ray Tune comes with several stopping mechanisms out of the box. For custom stopping behavior, you can +inherit from the :class:`Stopper ` class. + +Other stopping behaviors are described :ref:`in the user guide `. + +.. contents:: + :local: + :depth: 1 + + +.. _tune-stop-ref: + +Stopper (tune.Stopper) +---------------------- + +.. autoclass:: ray.tune.Stopper + :members: __call__, stop_all + +MaximumIterationStopper (tune.stopper.MaximumIterationStopper) +-------------------------------------------------------------- + +.. autoclass:: ray.tune.stopper.MaximumIterationStopper + +ExperimentPlateauStopper (tune.stopper.ExperimentPlateauStopper) +---------------------------------------------------------------- + +.. autoclass:: ray.tune.stopper.ExperimentPlateauStopper + +TrialPlateauStopper (tune.stopper.TrialPlateauStopper) +------------------------------------------------------ + +.. autoclass:: ray.tune.stopper.TrialPlateauStopper + +TimeoutStopper (tune.stopper.TimeoutStopper) +-------------------------------------------- + +.. autoclass:: ray.tune.stopper.TimeoutStopper diff --git a/doc/source/tune/user-guide.rst b/doc/source/tune/user-guide.rst index 1efa218b1..08fb4cba5 100644 --- a/doc/source/tune/user-guide.rst +++ b/doc/source/tune/user-guide.rst @@ -305,7 +305,9 @@ and passed to your trainable as a parameter. Stopping Trials --------------- -You can control when trials are stopped early by passing the ``stop`` argument to ``tune.run``. This argument takes either a dictionary or a function. +You can control when trials are stopped early by passing the ``stop`` argument to ``tune.run``. +This argument takes, a dictionary, a function, or a :class:`Stopper ` class +as an argument. If a dictionary is passed in, the keys may be any field in the return result of ``tune.report`` in the Function API or ``step()`` (including the results from ``step`` and auto-filled metrics). @@ -329,7 +331,7 @@ For more flexibility, you can pass in a function instead. If a function is passe tune.run(my_trainable, stop=stopper) -Finally, you can implement the ``Stopper`` abstract class for stopping entire experiments. For example, the following example stops all trials after the criteria is fulfilled by any individual trial, and prevents new ones from starting: +Finally, you can implement the :class:`Stopper ` abstract class for stopping entire experiments. For example, the following example stops all trials after the criteria is fulfilled by any individual trial, and prevents new ones from starting: .. code-block:: python @@ -352,7 +354,9 @@ Finally, you can implement the ``Stopper`` abstract class for stopping entire ex tune.run(my_trainable, stop=stopper) -Note that in the above example the currently running trials will not stop immediately but will do so once their current iterations are complete. See the :ref:`tune-stop-ref` documentation. +Note that in the above example the currently running trials will not stop immediately but will do so once their current iterations are complete. + +Ray Tune comes with a set of out-of-the-box stopper classes. See the :ref:`Stopper ` documentation. .. _tune-logging: diff --git a/doc/source/xgboost-ray.rst b/doc/source/xgboost-ray.rst index bc6c583fc..578f6b7a7 100644 --- a/doc/source/xgboost-ray.rst +++ b/doc/source/xgboost-ray.rst @@ -143,21 +143,3 @@ the `examples folder `__ * `HIGGS classification example with Parquet `__ (uses the same dataset) * `Test data classification `__ (uses a self-generated dataset) - -Package Reference ------------------ - - -Training/Validation -~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: ray.util.xgboost.RayParams - -.. autofunction:: ray.util.xgboost.train - -.. autofunction:: ray.util.xgboost.predict - -RayDMatrix -~~~~~~~~~~ - -.. autoclass:: ray.util.xgboost.RayDMatrix diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile index d4e559cdd..d878a8340 100644 --- a/docker/ray-ml/Dockerfile +++ b/docker/ray-ml/Dockerfile @@ -13,11 +13,10 @@ RUN sudo apt-get update \ libgtk2.0-dev \ zlib1g-dev \ libgl1-mesa-dev \ - && $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements.txt \ - && $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements_ml_docker.txt \ + && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements.txt \ + && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements_ml_docker.txt \ # Remove dataclasses & typing because they are included in Py3.7 && $HOME/anaconda3/bin/pip uninstall dataclasses typing -y \ && sudo rm requirements.txt && sudo rm requirements_ml_docker.txt \ - && sudo apt-get remove cmake gcc -y \ && sudo apt-get clean diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 0fdfb479a..119251c39 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -70,6 +70,7 @@ define_java_module( visibility = ["//visibility:public"], deps = [ ":io_ray_ray_api", + "@maven//:com_google_code_gson_gson", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", "@maven//:com_typesafe_config", @@ -134,27 +135,12 @@ filegroup( ], ) -native_java_binary("runtime", "raylet", "//:raylet") - -native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server") - -native_java_binary("runtime", "redis-server", "//:redis-server") - -native_java_binary("runtime", "gcs_server", "//:gcs_server") - -native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so") - native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so") filegroup( name = "java_native_deps", srcs = [ ":core_worker_library_java", - ":gcs_server", - ":libray_redis_module.so", - ":plasma_store_server", - ":raylet", - ":redis-server", ], ) @@ -252,13 +238,6 @@ genrule( WORK_DIR="$$(pwd)" rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars" cp -f $(location //java:ray_dist_deploy.jar) "$$WORK_DIR/python/ray/jars/ray_dist.jar" - chmod +w "$$WORK_DIR/python/ray/jars/ray_dist.jar" - zip -d "$$WORK_DIR/python/ray/jars/ray_dist.jar" \ - "native/*/gcs_server" \ - "native/*/libray_redis_module.so" \ - "native/*/plasma_store_server" \ - "native/*/raylet" \ - "native/*/redis-server" date > $@ """, local = 1, diff --git a/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java b/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java index e689cea00..620a40042 100644 --- a/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java +++ b/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java @@ -221,4 +221,12 @@ public interface RayRuntime { * @param id Id of the placement group. */ void removePlacementGroup(PlacementGroupId id); + + /** + * Wait for the placement group to be ready within the specified time. + * @param id Id of placement group. + * @param timeoutMs Timeout in milliseconds. + * @return True if the placement group is created. False otherwise. + */ + boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs); } diff --git a/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java b/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java index 3ccc48db1..bf4f733ad 100644 --- a/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java +++ b/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java @@ -28,16 +28,6 @@ public interface RuntimeContext { */ boolean wasCurrentActorRestarted(); - /** - * Get the raylet socket name. - */ - String getRayletSocketName(); - - /** - * Get the object store socket name. - */ - String getObjectStoreSocketName(); - /** * Return true if Ray is running in single-process mode, false if Ray is running in cluster mode. */ diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index a71b6c682..c09ec1912 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -39,6 +39,11 @@ ray-api ${project.version} + + com.google.code.gson + gson + 2.8.5 + com.google.guava guava diff --git a/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java b/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java index 2eae3b647..ac199fd95 100644 --- a/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java @@ -200,6 +200,11 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { return gcsClient.getAllPlacementGroupInfo(); } + @Override + public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) { + return taskSubmitter.waitPlacementGroupReady(id, timeoutMs); + } + @SuppressWarnings("unchecked") @Override public T getActorHandle(ActorId actorId) { diff --git a/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java b/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java index 5aa47cd58..3e4825458 100644 --- a/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java +++ b/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java @@ -16,7 +16,7 @@ public class DefaultRayRuntimeFactory implements RayRuntimeFactory { @Override public RayRuntime createRayRuntime() { - RayConfig rayConfig = RayConfig.getInstance(); + RayConfig rayConfig = RayConfig.create(); LoggingUtil.setupLogging(rayConfig); Logger logger = LoggerFactory.getLogger(DefaultRayRuntimeFactory.class); diff --git a/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java b/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java index 281677787..d7a686db8 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java @@ -57,7 +57,6 @@ public class RayDevRuntime extends AbstractRayRuntime { taskSubmitter = null; } taskExecutor = null; - RayConfig.reset(); } @Override diff --git a/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java index ee0f16e50..a9dbeeea5 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java @@ -5,10 +5,8 @@ import io.ray.api.BaseActorHandle; import io.ray.api.id.ActorId; import io.ray.api.id.JobId; import io.ray.api.id.UniqueId; -import io.ray.api.runtimecontext.NodeInfo; import io.ray.runtime.config.RayConfig; import io.ray.runtime.context.NativeWorkerContext; -import io.ray.runtime.exception.RayException; import io.ray.runtime.exception.RayIntentionalSystemExitException; import io.ray.runtime.gcs.GcsClient; import io.ray.runtime.gcs.GcsClientOptions; @@ -22,15 +20,12 @@ import io.ray.runtime.task.NativeTaskSubmitter; import io.ray.runtime.task.TaskExecutor; import io.ray.runtime.util.BinaryFileUtil; import io.ray.runtime.util.JniUtils; -import java.io.File; -import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +36,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime { private static final Logger LOGGER = LoggerFactory.getLogger(RayNativeRuntime.class); - private RunManager manager = null; + private boolean startRayHead = false; /** * In Java, GC runs in a standalone thread, and we can't control the exact @@ -52,124 +47,101 @@ public final class RayNativeRuntime extends AbstractRayRuntime { */ private final ReadWriteLock shutdownLock = new ReentrantReadWriteLock(); + public RayNativeRuntime(RayConfig rayConfig) { + super(rayConfig); + } - static { - LOGGER.debug("Loading native libraries."); - // Expose ray ABI symbols which may be depended by other shared - // libraries such as libstreaming_java.so. - // See BUILD.bazel:libcore_worker_library_java.so - final RayConfig rayConfig = RayConfig.getInstance(); - if (rayConfig.getRedisAddress() != null && rayConfig.workerMode == WorkerType.DRIVER) { - // Fetch session dir from GCS if this is a driver that is connecting to the existing GCS. + private void updateSessionDir() { + if (rayConfig.workerMode == WorkerType.DRIVER) { + // Fetch session dir from GCS if this is a driver. RedisClient client = new RedisClient(rayConfig.getRedisAddress(), rayConfig.redisPassword); final String sessionDir = client.get("session_dir", null); Preconditions.checkNotNull(sessionDir); rayConfig.setSessionDir(sessionDir); } - - JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true); - LOGGER.debug("Native libraries loaded."); - try { - FileUtils.forceMkdir(new File(rayConfig.logDir)); - } catch (IOException e) { - throw new RuntimeException("Failed to create the log directory.", e); - } } - public RayNativeRuntime(RayConfig rayConfig) { - super(rayConfig); - loadConfigFromGcs(rayConfig); - } - - private static void loadConfigFromGcs(RayConfig rayConfig) { - if (rayConfig.getRedisAddress() != null) { - GcsClient tempGcsClient = - new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword); - for (Map.Entry entry : - tempGcsClient.getInternalConfig().entrySet()) { - rayConfig.rayletConfigParameters.put(entry.getKey(), entry.getValue()); - } - - if (rayConfig.workerMode == WorkerType.DRIVER) { - // Keep this method logic in sync with `services.get_address_info_from_redis_helper` - int numRetries = 5; - int retryCount = 0; - boolean configLoaded = false; - while (retryCount++ < numRetries) { - for (NodeInfo nodeInfo : tempGcsClient.getAllNodeInfo()) { - if (rayConfig.nodeIp.equals(nodeInfo.nodeAddress) || - (nodeInfo.nodeAddress.equals("127.0.0.1") && - rayConfig.nodeIp.equals(rayConfig.getRedisAddress()))) { - rayConfig.objectStoreSocketName = nodeInfo.objectStoreSocketName; - rayConfig.rayletSocketName = nodeInfo.rayletSocketName; - rayConfig.nodeManagerPort = nodeInfo.nodeManagerPort; - configLoaded = true; - break; - } - } - if (!configLoaded) { - LOGGER.warn("Some processes that the driver needs to connect to have " + - "not registered with Redis, so retrying. Have you run " + - "'ray start' on this node?"); - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } else { - break; - } - } - if (!configLoaded) { - throw new RayException("Some processes that the driver needs to connect to have " + - "not registered with Redis. Have you run 'ray start' on this node?"); - } - } + private void loadConfigFromGcs() { + rayConfig.rayletConfigParameters.clear(); + for (Map.Entry entry : gcsClient.getInternalConfig().entrySet()) { + rayConfig.rayletConfigParameters.put(entry.getKey(), entry.getValue()); } } @Override public void start() { - if (rayConfig.getRedisAddress() == null) { - manager = new RunManager(rayConfig); - manager.startRayProcesses(true); + try { + if (rayConfig.workerMode == WorkerType.DRIVER && rayConfig.getRedisAddress() == null) { + // Set it to true before `RunManager.startRayHead` so `Ray.shutdown()` can still kill + // Ray processes even if `Ray.init()` failed. + startRayHead = true; + RunManager.startRayHead(rayConfig); + } + Preconditions.checkNotNull(rayConfig.getRedisAddress()); + + updateSessionDir(); + + // Expose ray ABI symbols which may be depended by other shared + // libraries such as libstreaming_java.so. + // See BUILD.bazel:libcore_worker_library_java.so + Preconditions.checkNotNull(rayConfig.sessionDir); + JniUtils.loadLibrary(rayConfig.sessionDir, BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true); + + if (rayConfig.workerMode == WorkerType.DRIVER) { + RunManager.getAddressInfoAndFillConfig(rayConfig); + } + + gcsClient = new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword); + + loadConfigFromGcs(); + + if (rayConfig.getJobId() == JobId.NIL) { + rayConfig.setJobId(gcsClient.nextJobId()); + } + int numWorkersPerProcess = + rayConfig.workerMode == WorkerType.DRIVER ? 1 : rayConfig.numWorkersPerProcess; + + byte[] serializedJobConfig = null; + if (rayConfig.workerMode == WorkerType.DRIVER) { + JobConfig.Builder jobConfigBuilder = + JobConfig.newBuilder() + .setNumJavaWorkersPerProcess(rayConfig.numWorkersPerProcess) + .addAllJvmOptions(rayConfig.jvmOptionsForJavaWorker) + .putAllWorkerEnv(rayConfig.workerEnv) + .addAllCodeSearchPath(rayConfig.codeSearchPath); + serializedJobConfig = jobConfigBuilder.build().toByteArray(); + } + + Map rayletConfigStringMap = new HashMap<>(); + for (Map.Entry entry : rayConfig.rayletConfigParameters.entrySet()) { + rayletConfigStringMap.put(entry.getKey(), entry.getValue().toString()); + } + + nativeInitialize(rayConfig.workerMode.getNumber(), + rayConfig.nodeIp, rayConfig.getNodeManagerPort(), + rayConfig.workerMode == WorkerType.DRIVER ? System.getProperty("user.dir") : "", + rayConfig.objectStoreSocketName, rayConfig.rayletSocketName, + (rayConfig.workerMode == WorkerType.DRIVER ? rayConfig.getJobId() : JobId.NIL).getBytes(), + new GcsClientOptions(rayConfig), numWorkersPerProcess, + rayConfig.logDir, rayletConfigStringMap, serializedJobConfig); + + taskExecutor = new NativeTaskExecutor(this); + workerContext = new NativeWorkerContext(); + objectStore = new NativeObjectStore(workerContext, shutdownLock); + taskSubmitter = new NativeTaskSubmitter(); + + LOGGER.debug("RayNativeRuntime started with store {}, raylet {}", + rayConfig.objectStoreSocketName, rayConfig.rayletSocketName); + } catch (Exception e) { + if (startRayHead) { + try { + RunManager.stopRay(); + } catch (Exception e2) { + // Ignore + } + } + throw e; } - - gcsClient = new GcsClient(rayConfig.getRedisAddress(), rayConfig.redisPassword); - - if (rayConfig.getJobId() == JobId.NIL) { - rayConfig.setJobId(gcsClient.nextJobId()); - } - int numWorkersPerProcess = - rayConfig.workerMode == WorkerType.DRIVER ? 1 : rayConfig.numWorkersPerProcess; - - byte[] serializedJobConfig = null; - if (rayConfig.workerMode == WorkerType.DRIVER) { - JobConfig.Builder jobConfigBuilder = - JobConfig.newBuilder() - .setNumJavaWorkersPerProcess(rayConfig.numWorkersPerProcess) - .addAllJvmOptions(rayConfig.jvmOptionsForJavaWorker) - .putAllWorkerEnv(rayConfig.workerEnv) - .addAllCodeSearchPath(rayConfig.codeSearchPath); - serializedJobConfig = jobConfigBuilder.build().toByteArray(); - } - - // TODO(qwang): Get object_store_socket_name and raylet_socket_name from Redis. - nativeInitialize(rayConfig.workerMode.getNumber(), - rayConfig.nodeIp, rayConfig.getNodeManagerPort(), - rayConfig.workerMode == WorkerType.DRIVER ? System.getProperty("user.dir") : "", - rayConfig.objectStoreSocketName, rayConfig.rayletSocketName, - (rayConfig.workerMode == WorkerType.DRIVER ? rayConfig.getJobId() : JobId.NIL).getBytes(), - new GcsClientOptions(rayConfig), numWorkersPerProcess, - rayConfig.logDir, rayConfig.rayletConfigParameters, serializedJobConfig); - - taskExecutor = new NativeTaskExecutor(this); - workerContext = new NativeWorkerContext(); - objectStore = new NativeObjectStore(workerContext, shutdownLock); - taskSubmitter = new NativeTaskSubmitter(); - - LOGGER.debug("RayNativeRuntime started with store {}, raylet {}", - rayConfig.objectStoreSocketName, rayConfig.rayletSocketName); } @Override @@ -183,27 +155,21 @@ public final class RayNativeRuntime extends AbstractRayRuntime { try { if (rayConfig.workerMode == WorkerType.DRIVER) { nativeShutdown(); - if (null != manager) { - manager.cleanup(); - manager = null; + if (startRayHead) { + startRayHead = false; + RunManager.stopRay(); } } if (null != gcsClient) { gcsClient.destroy(); gcsClient = null; } - RayConfig.reset(); LOGGER.debug("RayNativeRuntime shutdown"); } finally { writeLock.unlock(); } } - // For test purpose only - public RunManager getRunManager() { - return manager; - } - @Override public void setResource(String resourceName, double capacity, UniqueId nodeId) { Preconditions.checkArgument(Double.compare(capacity, 0) >= 0); diff --git a/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java index e7f5cddb7..f82a7a124 100644 --- a/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java @@ -2,7 +2,6 @@ package io.ray.runtime.config; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.typesafe.config.Config; import com.typesafe.config.ConfigException; @@ -12,17 +11,14 @@ import com.typesafe.config.ConfigValue; import io.ray.api.id.JobId; import io.ray.runtime.generated.Common.WorkerType; import io.ray.runtime.util.NetworkUtil; -import io.ray.runtime.util.ResourceUtil; import java.io.File; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; /** * Configurations of Ray runtime. @@ -33,13 +29,6 @@ public class RayConfig { public static final String DEFAULT_CONFIG_FILE = "ray.default.conf"; public static final String CUSTOM_CONFIG_FILE = "ray.conf"; - private static final Random RANDOM = new Random(); - - private static final DateTimeFormatter DATE_TIME_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); - - private static final String DEFAULT_TEMP_DIR = "/tmp/ray"; - private Config config; /** @@ -48,54 +37,25 @@ public class RayConfig { public final String nodeIp; public final WorkerType workerMode; public final RunMode runMode; - public final Map resources; private JobId jobId; public String sessionDir; public String logDir; - public final List libraryPath; - public final List classpath; - public final List jvmParameters; private String redisAddress; - private String redisIp; - private Integer redisPort; - public final int headRedisPort; - public final int[] redisShardPorts; - public final int numberRedisShards; - public final String headRedisPassword; public final String redisPassword; // RPC socket name of object store. public String objectStoreSocketName; - public final Long objectStoreSize; // RPC socket name of Raylet. public String rayletSocketName; // Listening port for node manager. public int nodeManagerPort; - public final Map rayletConfigParameters; + public final Map rayletConfigParameters; - public List codeSearchPath; - public final String pythonWorkerCommand; + public final List codeSearchPath; - private static volatile RayConfig instance = null; - - public static RayConfig getInstance() { - if (instance == null) { - synchronized (RayConfig.class) { - if (instance == null) { - instance = RayConfig.create(); - } - } - } - return instance; - } - - public static void reset() { - synchronized (RayConfig.class) { - instance = null; - } - } + public final List headArgs; public final int numWorkersPerProcess; @@ -140,15 +100,6 @@ public class RayConfig { } else { nodeIp = NetworkUtil.getIpAddress(null); } - // Resources. - resources = ResourceUtil.getResourcesMapFromString( - config.getString("ray.resources")); - if (isDriver) { - if (!resources.containsKey("CPU")) { - int numCpu = Runtime.getRuntime().availableProcessors(); - resources.put("CPU", numCpu * 1.0); - } - } // Job id. String jobId = config.getString("ray.job.id"); if (!jobId.isEmpty()) { @@ -168,25 +119,16 @@ public class RayConfig { } } workerEnv = workerEnvBuilder.build(); - updateSessionDir(); - // Object store configurations. - objectStoreSize = config.getBytes("ray.object-store.size"); + updateSessionDir(null); - // Library path. - libraryPath = config.getStringList("ray.library.path"); - // Custom classpath. - classpath = config.getStringList("ray.classpath"); - // Custom worker jvm parameters. - if (config.hasPath("ray.worker.jvm-parameters")) { - jvmParameters = config.getStringList("ray.worker.jvm-parameters"); - } else { - jvmParameters = ImmutableList.of(); + // Object store socket name. + if (config.hasPath("ray.object-store.socket-name")) { + objectStoreSocketName = config.getString("ray.object-store.socket-name"); } - if (config.hasPath("ray.worker.python-command")) { - pythonWorkerCommand = config.getString("ray.worker.python-command"); - } else { - pythonWorkerCommand = null; + // Raylet socket name. + if (config.hasPath("ray.raylet.socket-name")) { + rayletSocketName = config.getString("ray.raylet.socket-name"); } // Redis configurations. @@ -198,17 +140,6 @@ public class RayConfig { this.redisAddress = null; } - if (config.hasPath("ray.redis.head-port")) { - headRedisPort = config.getInt("ray.redis.head-port"); - } else { - headRedisPort = NetworkUtil.getUnusedPort(); - } - numberRedisShards = config.getInt("ray.redis.shard-number"); - redisShardPorts = new int[numberRedisShards]; - for (int i = 0; i < numberRedisShards; i++) { - redisShardPorts[i] = NetworkUtil.getUnusedPort(); - } - headRedisPassword = config.getString("ray.redis.head-password"); redisPassword = config.getString("ray.redis.password"); // Raylet node manager port. if (config.hasPath("ray.raylet.node-manager-port")) { @@ -216,7 +147,6 @@ public class RayConfig { } else { Preconditions.checkState(workerMode != WorkerType.WORKER, "Worker started by raylet should accept the node manager port from raylet."); - nodeManagerPort = NetworkUtil.getUnusedPort(); } // Raylet parameters. @@ -224,39 +154,33 @@ public class RayConfig { Config rayletConfig = config.getConfig("ray.raylet.config"); for (Map.Entry entry : rayletConfig.entrySet()) { Object value = entry.getValue().unwrapped(); - rayletConfigParameters.put(entry.getKey(), value == null ? "" : value.toString()); + if (value != null) { + if (value instanceof String) { + String valueString = (String) value; + Boolean booleanValue = BooleanUtils.toBooleanObject(valueString); + if (booleanValue != null) { + value = booleanValue; + } else if (NumberUtils.isParsable(valueString)) { + value = NumberUtils.createNumber(valueString); + } + } + rayletConfigParameters.put(entry.getKey(), value); + } } // Job code search path. + String codeSearchPathString = null; if (config.hasPath("ray.job.code-search-path")) { - codeSearchPath = Arrays.asList( - config.getString("ray.job.code-search-path").split(":")); - } else { - codeSearchPath = Collections.emptyList(); + codeSearchPathString = config.getString("ray.job.code-search-path"); } + if (StringUtils.isEmpty(codeSearchPathString)) { + codeSearchPathString = System.getProperty("java.class.path"); + } + codeSearchPath = Arrays.asList(codeSearchPathString.split(":")); - boolean enableMultiTenancy; - if (config.hasPath("ray.raylet.config.enable_multi_tenancy")) { - enableMultiTenancy = - Boolean.valueOf(config.getString("ray.raylet.config.enable_multi_tenancy")); - } else { - String envString = System.getenv("RAY_ENABLE_MULTI_TENANCY"); - if (StringUtils.isNotBlank(envString)) { - enableMultiTenancy = "1".equals(envString); - } else { - enableMultiTenancy = true; // Default value - } - } + numWorkersPerProcess = config.getInt("ray.job.num-java-workers-per-process"); - if (!enableMultiTenancy) { - if (!isDriver) { - numWorkersPerProcess = config.getInt("ray.raylet.config.num_workers_per_process_java"); - } else { - numWorkersPerProcess = 1; // Actually this value isn't used in RayNativeRuntime. - } - } else { - numWorkersPerProcess = config.getInt("ray.job.num-java-workers-per-process"); - } + headArgs = config.getStringList("ray.head-args"); // Validate config. validate(); @@ -267,24 +191,12 @@ public class RayConfig { Preconditions.checkState(this.redisAddress == null, "Redis address was already set"); this.redisAddress = redisAddress; - String[] ipAndPort = redisAddress.split(":"); - Preconditions.checkArgument(ipAndPort.length == 2, "Invalid redis address."); - this.redisIp = ipAndPort[0]; - this.redisPort = Integer.parseInt(ipAndPort[1]); } public String getRedisAddress() { return redisAddress; } - public String getRedisIp() { - return redisIp; - } - - public Integer getRedisPort() { - return redisPort; - } - public void setJobId(JobId jobId) { this.jobId = jobId; } @@ -298,11 +210,7 @@ public class RayConfig { } public void setSessionDir(String sessionDir) { - this.sessionDir = sessionDir; - } - - public String getSessionDir() { - return sessionDir; + updateSessionDir(sessionDir); } public Config getInternalConfig() { @@ -312,7 +220,8 @@ public class RayConfig { /** * Renders the config value as a HOCON string. */ - public String render() { + @Override + public String toString() { // These items might be dynamically generated or mutated at runtime. // Explicitly include them. Map dynamic = new HashMap<>(); @@ -321,24 +230,19 @@ public class RayConfig { dynamic.put("ray.object-store.socket-name", objectStoreSocketName); dynamic.put("ray.raylet.node-manager-port", nodeManagerPort); dynamic.put("ray.address", redisAddress); - dynamic.put("ray.job.code-search-path", codeSearchPath); Config toRender = ConfigFactory.parseMap(dynamic).withFallback(config); return toRender.root().render(ConfigRenderOptions.concise()); } - private void updateSessionDir() { + private void updateSessionDir(String sessionDir) { // session dir - if (workerMode == WorkerType.DRIVER) { - final int minBound = 100000; - final int maxBound = 999999; - final String sessionName = String.format("session_%s_%d", DATE_TIME_FORMATTER.format( - LocalDateTime.now()), RANDOM.nextInt(maxBound - minBound) + minBound); - sessionDir = String.format("%s/%s", DEFAULT_TEMP_DIR, sessionName); - } else if (workerMode == WorkerType.WORKER) { - sessionDir = removeTrailingSlash(config.getString("ray.session-dir")); - } else { - throw new RuntimeException("Unknown worker type."); + if (config.hasPath("ray.session-dir")) { + sessionDir = config.getString("ray.session-dir"); } + if (sessionDir != null) { + sessionDir = removeTrailingSlash(sessionDir); + } + this.sessionDir = sessionDir; // Log dir. String localLogDir = null; @@ -350,34 +254,6 @@ public class RayConfig { } else { logDir = localLogDir; } - - // Object store socket name. - String localObjectStoreSocketName = null; - if (config.hasPath("ray.object-store.socket-name")) { - localObjectStoreSocketName = config.getString("ray.object-store.socket-name"); - } - if (Strings.isNullOrEmpty(localObjectStoreSocketName)) { - objectStoreSocketName = String.format("%s/sockets/object_store", sessionDir); - } else { - objectStoreSocketName = localObjectStoreSocketName; - } - - // Raylet socket name. - String localRayletSocketName = null; - if (config.hasPath("ray.raylet.socket-name")) { - localRayletSocketName = config.getString("ray.raylet.socket-name"); - } - if (Strings.isNullOrEmpty(localRayletSocketName)) { - rayletSocketName = String.format("%s/sockets/raylet", sessionDir); - } else { - rayletSocketName = localRayletSocketName; - } - - } - - @Override - public String toString() { - return render(); } /** diff --git a/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java b/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java index a96c11ba6..fc36fac3d 100644 --- a/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java @@ -43,16 +43,6 @@ public class RuntimeContextImpl implements RuntimeContext { return runtime.getGcsClient().wasCurrentActorRestarted(getCurrentActorId()); } - @Override - public String getRayletSocketName() { - return runtime.getRayConfig().rayletSocketName; - } - - @Override - public String getObjectStoreSocketName() { - return runtime.getRayConfig().objectStoreSocketName; - } - @Override public boolean isSingleProcess() { return RunMode.SINGLE_PROCESS == runtime.getRayConfig().runMode; diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java index 379bd20d8..a21306c92 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java @@ -1,5 +1,6 @@ package io.ray.runtime.gcs; +import com.google.common.base.Preconditions; import io.ray.runtime.config.RayConfig; /** @@ -11,8 +12,10 @@ public class GcsClientOptions { public String password; public GcsClientOptions(RayConfig rayConfig) { - ip = rayConfig.getRedisIp(); - port = rayConfig.getRedisPort(); + String[] ipAndPort = rayConfig.getRedisAddress().split(":"); + Preconditions.checkArgument(ipAndPort.length == 2, "Invalid redis address."); + ip = ipAndPort[0]; + port = Integer.parseInt(ipAndPort[1]); password = rayConfig.redisPassword; } } diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java index 23516e547..663e62a77 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java @@ -82,8 +82,7 @@ public class GlobalStateAccessor { public byte[] getPlacementGroupInfo(PlacementGroupId placementGroupId) { synchronized (GlobalStateAccessor.class) { - Preconditions.checkNotNull(placementGroupId, - "PlacementGroupId can't be null when get placement group info."); + validateGlobalStateAccessorPointer(); return nativeGetPlacementGroupInfo(globalStateAccessorNativePointer, placementGroupId.getBytes()); } diff --git a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java index 633bad98c..96f499e60 100644 --- a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java @@ -1,5 +1,6 @@ package io.ray.runtime.placementgroup; +import io.ray.api.Ray; import io.ray.api.id.PlacementGroupId; import io.ray.api.placementgroup.PlacementGroup; import io.ray.api.placementgroup.PlacementGroupState; @@ -49,6 +50,15 @@ public class PlacementGroupImpl implements PlacementGroup { return state; } + /** + * Wait for the placement group to be ready within the specified time. + * @param timeoutSeconds Timeout in seconds. + * @return True if the placement group is created. False otherwise. + */ + public boolean wait(int timeoutSeconds) { + return Ray.internal().waitPlacementGroupReady(id, timeoutSeconds); + } + /** * A help class for create the placement group. */ diff --git a/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java index 5823fccf5..b6ed41f7f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java @@ -1,31 +1,20 @@ package io.ray.runtime.runner; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.ray.runtime.config.RayConfig; -import io.ray.runtime.util.BinaryFileUtil; -import io.ray.runtime.util.ResourceUtil; -import java.io.File; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import redis.clients.jedis.Jedis; /** * Ray service management on one box. @@ -34,97 +23,71 @@ public class RunManager { private static final Logger LOGGER = LoggerFactory.getLogger(RunManager.class); - private static final String WORKER_CLASS = "io.ray.runtime.runner.worker.DefaultWorker"; - - private static final String SESSION_LATEST = "session_latest"; - - private RayConfig rayConfig; - - private List> processes; - - private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1; - - public RunManager(RayConfig rayConfig) { - this.rayConfig = rayConfig; - processes = new ArrayList<>(); - createTempDirs(); - } - - public void cleanup() { - // Terminate the processes in the reversed order of creating them. - // Because raylet needs to exit before object store, otherwise it - // cannot exit gracefully. - - for (int i = processes.size() - 1; i >= 0; --i) { - Pair pair = processes.get(i); - terminateProcess(pair.getLeft(), pair.getRight()); - } - } - - public void terminateProcess(String name, Process p) { - int numAttempts = 0; - while (p.isAlive()) { - if (numAttempts == 0) { - LOGGER.debug("Terminating process {}.", name); - p.destroy(); - } else { - LOGGER.debug("Terminating process {} forcibly.", name); - p.destroyForcibly(); - } - try { - p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOGGER.warn("Got InterruptedException while waiting for process {}" + - " to be terminated.", name); - } - numAttempts++; - } - LOGGER.debug("Process {} is now terminated.", name); - } + private static final Pattern pattern = Pattern.compile("--address='([^']+)'"); /** - * Get processes by name. For test purposes only. + * Start the head node. */ - public List getProcesses(String name) { - return processes.stream().filter(pair -> pair.getLeft().equals(name)).map(Pair::getRight) - .collect(Collectors.toList()); - } - - private void createTempDirs() { + public static void startRayHead(RayConfig rayConfig) { + LOGGER.debug("Starting ray runtime @ {}.", rayConfig.nodeIp); + List command = new ArrayList<>(); + command.add("ray"); + command.add("start"); + command.add("--head"); + command.add("--redis-password"); + command.add(rayConfig.redisPassword); + command.add("--system-config=" + new Gson().toJson(rayConfig.rayletConfigParameters)); + command.addAll(rayConfig.headArgs); + String output; try { - FileUtils.forceMkdir(new File(rayConfig.logDir)); - FileUtils.forceMkdir(new File(rayConfig.rayletSocketName).getParentFile()); - FileUtils.forceMkdir(new File(rayConfig.objectStoreSocketName).getParentFile()); - - // Remove session_latest first, and then create a new symbolic link for session_latest. - final String parentOfSessionDir = new File(rayConfig.sessionDir).getParent(); - final File sessionLatest = new File( - String.format("%s/%s", parentOfSessionDir, SESSION_LATEST)); - if (sessionLatest.exists()) { - sessionLatest.delete(); - } - Files.createSymbolicLink( - Paths.get(sessionLatest.getAbsolutePath()), - Paths.get(rayConfig.sessionDir)); - } catch (IOException e) { - LOGGER.error("Couldn't create temp directories.", e); - throw new RuntimeException(e); + output = runCommand(command); + } catch (Exception e) { + throw new RuntimeException("Failed to start Ray runtime.", e); } + Matcher matcher = pattern.matcher(output); + if (matcher.find()) { + String redisAddress = matcher.group(1); + rayConfig.setRedisAddress(redisAddress); + } else { + throw new RuntimeException("Redis address is not found. output: " + output); + } + LOGGER.info("Ray runtime started @ {}.", rayConfig.nodeIp); } /** - * @return Log files for stdout and stderr. + * Stop ray. */ - private Pair getLogFiles(String logDir, String processName) { - int suffixIndex = 0; - while (true) { - String suffix = suffixIndex == 0 ? "" : "." + suffixIndex; - File stdout = new File(String.format("%s/%s%s.out", logDir, suffix, processName)); - File stderr = new File(String.format("%s/%s%s.err", logDir, suffix, processName)); - if (!stdout.exists() && !stderr.exists()) { - return ImmutablePair.of(stdout, stderr); - } - suffixIndex += 1; + public static void stopRay() { + List command = new ArrayList<>(); + command.add("ray"); + command.add("stop"); + command.add("--force"); + + try { + runCommand(command); + } catch (Exception e) { + throw new RuntimeException("Failed to stop ray.", e); + } + } + + public static void getAddressInfoAndFillConfig(RayConfig rayConfig) { + // NOTE(kfstorm): This method depends on an internal Python API of ray to get the + // address info of the local node. + String script = String.format("import ray;" + + " print(ray._private.services.get_address_info_from_redis(" + + "'%s', '%s', redis_password='%s', no_warning=True))", + rayConfig.getRedisAddress(), rayConfig.nodeIp, rayConfig.redisPassword); + List command = Arrays.asList("python", "-c", script); + + String output = null; + try { + output = runCommand(command); + JsonObject addressInfo = new JsonParser().parse(output).getAsJsonObject(); + rayConfig.rayletSocketName = addressInfo.get("raylet_socket_name").getAsString(); + rayConfig.objectStoreSocketName = addressInfo.get("object_store_address").getAsString(); + rayConfig.nodeManagerPort = addressInfo.get("node_manager_port").getAsInt(); + } catch (Exception e) { + throw new RuntimeException("Failed to get address info. Output: " + output, e); } } @@ -132,284 +95,22 @@ public class RunManager { * Start a process. * * @param command The command to start the process with. - * @param env Environment variables. - * @param name Process name. */ - private void startProcess(List command, Map env, String name) { + private static String runCommand(List command) throws IOException, InterruptedException { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Starting process {} with command: {}", name, - Joiner.on(" ").join(command)); + LOGGER.debug("Starting process with command: {}", Joiner.on(" ").join(command)); } - ProcessBuilder builder = new ProcessBuilder(command); - - String stdout = ""; - String stderr = ""; - // Set stdout and stderr paths. - Pair logFiles = getLogFiles(rayConfig.logDir, name); - builder.redirectOutput(logFiles.getLeft()); - builder.redirectError(logFiles.getRight()); - - // Set environment variables. - if (env != null && !env.isEmpty()) { - builder.environment().putAll(env); - } - - Process p; - try { - p = builder.start(); - } catch (IOException e) { - LOGGER.error("Failed to start process " + name, e); - throw new RuntimeException("Failed to start process " + name, e); - } - // Wait 1000 ms and check whether the process is alive. - try { - TimeUnit.MILLISECONDS.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (!p.isAlive()) { - String message = String.format("Failed to start %s. Exit code: %d.", - name, p.exitValue()); - message += String.format(" Logs are redirected to %s and %s.", stdout, stderr); - throw new RuntimeException(message); - } - processes.add(Pair.of(name, p)); - if (LOGGER.isDebugEnabled()) { - String message = String.format("%s process started.", name); - message += String.format(" Logs are redirected to %s and %s.", stdout, stderr); - LOGGER.debug(message); + ProcessBuilder builder = new ProcessBuilder(command).redirectErrorStream(true); + Process p = builder.start(); + String output = IOUtils.toString(p.getInputStream(), Charset.defaultCharset()); + p.waitFor(); + if (p.exitValue() != 0) { + String sb = "The exit value of the process is " + p.exitValue() + + ". Command: " + Joiner.on(" ").join(command) + "\n" + + "output:\n" + output; + throw new RuntimeException(sb); } + return output; } - - /** - * Start all Ray processes on this node. - * - * @param isHead Whether this node is the head node. If true, redis server will be started. - */ - public void startRayProcesses(boolean isHead) { - LOGGER.debug("Starting ray runtime @ {}.", rayConfig.nodeIp); - try { - if (isHead) { - startGcs(); - } - startObjectStore(); - startRaylet(isHead); - LOGGER.info("Ray runtime started @ {}.", rayConfig.nodeIp); - } catch (Exception e) { - // Clean up started processes. - cleanup(); - LOGGER.error("Failed to start ray runtime.", e); - throw new RuntimeException("Failed to start ray runtime.", e); - } - } - - private void startGcs() { - // start primary redis - String primary = startRedisInstance(rayConfig.nodeIp, - rayConfig.headRedisPort, rayConfig.headRedisPassword, null); - rayConfig.setRedisAddress(primary); - try (Jedis client = new Jedis("127.0.0.1", rayConfig.headRedisPort)) { - if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { - client.auth(rayConfig.headRedisPassword); - } - client.set("UseRaylet", "1"); - // Set job counter to compute job id. - client.set("JobCounter", "0"); - // Register the number of Redis shards in the primary shard, so that clients - // know how many redis shards to expect under RedisShards. - client.set("NumRedisShards", Integer.toString(rayConfig.numberRedisShards)); - // Set session dir for this cluster, so that the drivers which connected to this - // cluster will fetch this session dir as its self's session dir. - client.set("session_dir", rayConfig.getSessionDir()); - // start redis shards - for (int i = 0; i < rayConfig.numberRedisShards; i++) { - String shard = startRedisInstance(rayConfig.nodeIp, - rayConfig.redisShardPorts[i], rayConfig.headRedisPassword, i); - client.rpush("RedisShards", shard); - } - } - - // start gcs server - String redisPasswordOption = ""; - if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { - redisPasswordOption = rayConfig.headRedisPassword; - } - - // See `src/ray/gcs/gcs_server/gcs_server_main.cc` for the meaning of each parameter. - final File gcsServerFile = BinaryFileUtil.getNativeFile( - rayConfig.sessionDir, BinaryFileUtil.GCS_SERVER_BINARY_NAME); - Preconditions.checkState(gcsServerFile.setExecutable(true)); - List command = ImmutableList.of( - gcsServerFile.getAbsolutePath(), - String.format("--redis_address=%s", rayConfig.getRedisIp()), - String.format("--redis_port=%d", rayConfig.getRedisPort()), - String.format("--config_list=%s", - rayConfig.rayletConfigParameters.entrySet().stream() - .map(entry -> entry.getKey() + "," + entry.getValue()).collect(Collectors - .joining(","))), - String.format("--redis_password=%s", redisPasswordOption) - ); - startProcess(command, null, "gcs_server"); - } - - private String startRedisInstance(String ip, int port, String password, Integer shard) { - final File redisServerFile = BinaryFileUtil.getNativeFile( - rayConfig.sessionDir, BinaryFileUtil.REDIS_SERVER_BINARY_NAME); - Preconditions.checkState(redisServerFile.setExecutable(true)); - // The redis module file. - File redisModule = BinaryFileUtil.getNativeFile( - rayConfig.sessionDir, BinaryFileUtil.REDIS_MODULE_LIBRARY_NAME); - Preconditions.checkState(redisModule.setExecutable(true)); - List command = Lists.newArrayList( - // The redis-server executable file. - redisServerFile.getAbsolutePath(), - "--protected-mode", - "no", - "--port", - String.valueOf(port), - "--loglevel", - "warning", - "--loadmodule", - // The redis module file. - redisModule.getAbsolutePath() - ); - - if (!Strings.isNullOrEmpty(password)) { - command.add("--requirepass "); - command.add(password); - } - - String name = shard == null ? "redis" : "redis-shard_" + shard; - startProcess(command, null, name); - - try (Jedis client = new Jedis("127.0.0.1", port)) { - if (!Strings.isNullOrEmpty(password)) { - client.auth(password); - } - - // Configure Redis to only generate notifications for the export keys. - client.configSet("notify-keyspace-events", "Kl"); - // Put a time stamp in Redis to indicate when it was started. - client.set("redis_start_time", LocalDateTime.now().toString()); - } - - return ip + ":" + port; - } - - private void startRaylet(boolean isHead) throws IOException { - int hardwareConcurrency = Runtime.getRuntime().availableProcessors(); - int maximumStartupConcurrency = Math.max(1, - Math.min(rayConfig.resources.getOrDefault("CPU", 0.0).intValue(), hardwareConcurrency)); - - String redisPasswordOption = ""; - if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { - redisPasswordOption = rayConfig.headRedisPassword; - } - - // See `src/ray/raylet/main.cc` for the meaning of each parameter. - final File rayletFile = BinaryFileUtil.getNativeFile( - rayConfig.sessionDir, BinaryFileUtil.RAYLET_BINARY_NAME); - Preconditions.checkState(rayletFile.setExecutable(true)); - List command = ImmutableList.of( - rayletFile.getAbsolutePath(), - String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName), - String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName), - String.format("--object_manager_port=%d", 0), // The object manager port. - // The node manager port. - String.format("--node_manager_port=%d", rayConfig.getNodeManagerPort()), - String.format("--node_ip_address=%s", rayConfig.nodeIp), - String.format("--redis_address=%s", rayConfig.getRedisIp()), - String.format("--redis_port=%d", rayConfig.getRedisPort()), - String.format("--num_initial_workers=%d", 0), // number of initial workers - String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency), - String.format("--static_resource_list=%s", - ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), - String.format("--config_list=%s", rayConfig.rayletConfigParameters.entrySet().stream() - .map(entry -> entry.getKey() + "," + entry.getValue()) - .collect(Collectors.joining(","))), - String.format("--python_worker_command=%s", buildPythonWorkerCommand()), - String.format("--java_worker_command=%s", buildWorkerCommand()), - String.format("--redis_password=%s", redisPasswordOption), - isHead ? "--head_node" : "" - ); - - startProcess(command, null, "raylet"); - } - - private String concatPath(Stream stream) { - // TODO (hchen): Right now, raylet backend doesn't support worker command with spaces. - // Thus, we have to drop some some paths until that is fixed. - return stream.filter(s -> !s.contains(" ")).collect(Collectors.joining(":")); - } - - private String buildWorkerCommand() throws IOException { - List cmd = new ArrayList<>(); - cmd.add("java"); - cmd.add("-classpath"); - - // Generate classpath based on current classpath + user-defined classpath. - String classpath = concatPath(Stream.concat( - rayConfig.classpath.stream(), - Stream.of(System.getProperty("java.class.path").split(":")) - )); - cmd.add(classpath); - - // Write current config to a file, and set the file path as Java worker's config file. - // This allows users to set worker config by setting driver's system properties. - File workerConfigFile = new File(rayConfig.sessionDir + "/java_worker.conf"); - FileUtils.write(workerConfigFile, rayConfig.render(), Charset.defaultCharset()); - cmd.add("-Dray.config-file=" + workerConfigFile.getAbsolutePath()); - if (!rayConfig.codeSearchPath.isEmpty()) { - cmd.add("-Dray.job.code-search-path=" + - String.join(":", rayConfig.codeSearchPath)); - } - cmd.add("RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER"); - - cmd.addAll(rayConfig.jvmParameters); - - // jvm options - cmd.add("RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER"); - - // Main class - cmd.add(WORKER_CLASS); - String command = Joiner.on(" ").join(cmd); - LOGGER.debug("Worker command is: {}", command); - return command; - } - - private void startObjectStore() { - final File objectStoreFile = BinaryFileUtil.getNativeFile( - rayConfig.sessionDir, BinaryFileUtil.PLASMA_STORE_SERVER_BINARY_NAME); - Preconditions.checkState(objectStoreFile.setExecutable(true)); - List command = ImmutableList.of( - // The plasma store executable file. - objectStoreFile.getAbsolutePath(), - "-s", - rayConfig.objectStoreSocketName, - "-m", - rayConfig.objectStoreSize.toString() - ); - startProcess(command, null, "plasma_store"); - } - - - private String buildPythonWorkerCommand() { - // disable python worker start from raylet, which starts from java - if (rayConfig.pythonWorkerCommand == null) { - return ""; - } - - List cmd = new ArrayList<>(); - cmd.add(rayConfig.pythonWorkerCommand); - cmd.add("--node-ip-address=" + rayConfig.nodeIp); - cmd.add("--object-store-name=" + rayConfig.objectStoreSocketName); - cmd.add("--raylet-name=" + rayConfig.rayletSocketName); - cmd.add("--address=" + rayConfig.getRedisAddress()); - - String command = cmd.stream().collect(Collectors.joining(" ")); - LOGGER.debug("python worker command: {}", command); - return command; - } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskSubmitter.java b/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskSubmitter.java index 53d7d2ae2..1e9304ade 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskSubmitter.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskSubmitter.java @@ -240,6 +240,11 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { placementGroups.remove(id); } + @Override + public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) { + return true; + } + @Override public BaseActorHandle getActor(ActorId actorId) { return actorHandles.get(actorId).copy(); diff --git a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java index dd2def600..15e193360 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java @@ -91,6 +91,11 @@ public class NativeTaskSubmitter implements TaskSubmitter { nativeRemovePlacementGroup(id.getBytes()); } + @Override + public boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs) { + return nativeWaitPlacementGroupReady(id.getBytes(), timeoutMs); + } + private static native List nativeSubmitTask(FunctionDescriptor functionDescriptor, int functionDescriptorHash, List args, int numReturns, CallOptions callOptions); @@ -107,4 +112,6 @@ public class NativeTaskSubmitter implements TaskSubmitter { private static native void nativeRemovePlacementGroup(byte[] placementGroupId); + private static native boolean nativeWaitPlacementGroupReady(byte[] placementGroupId, + int timeoutMs); } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java b/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java index 5c172caf9..17a8d34f9 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java @@ -68,6 +68,14 @@ public interface TaskSubmitter { */ void removePlacementGroup(PlacementGroupId id); + /** + * Wait for the placement group to be ready within the specified time. + * @param id Id of placement group. + * @param timeoutMs Timeout in milliseconds. + * @return True if the placement group is created. False otherwise. + */ + boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs); + BaseActorHandle getActor(ActorId actorId); } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java index f8a9eb34b..b73e7826d 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java @@ -12,15 +12,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; public class BinaryFileUtil { - public static final String REDIS_SERVER_BINARY_NAME = "redis-server"; - - public static final String GCS_SERVER_BINARY_NAME = "gcs_server"; - - public static final String PLASMA_STORE_SERVER_BINARY_NAME = "plasma_store_server"; - - public static final String RAYLET_BINARY_NAME = "raylet"; - - public static final String REDIS_MODULE_LIBRARY_NAME = "libray_redis_module.so"; public static final String CORE_WORKER_JAVA_LIBRARY = "core_worker_library_java"; diff --git a/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java b/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java index df49c008b..f63cf72d3 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java @@ -2,8 +2,9 @@ package io.ray.runtime.util; import com.google.common.collect.Sets; import com.sun.jna.NativeLibrary; -import io.ray.runtime.config.RayConfig; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,17 +12,7 @@ import org.slf4j.LoggerFactory; public class JniUtils { private static final Logger LOGGER = LoggerFactory.getLogger(JniUtils.class); private static Set loadedLibs = Sets.newHashSet(); - - /** - * Loads the native library specified by the libraryName argument. - * The libraryName argument must not contain any platform specific - * prefix, file extension or path. - * - * @param libraryName the name of the library. - */ - public static synchronized void loadLibrary(String libraryName) { - loadLibrary(libraryName, false); - } + private static String defaultDestDir; /** * Loads the native library specified by the libraryName argument. @@ -29,15 +20,51 @@ public class JniUtils { * prefix, file extension or path. * * @param libraryName the name of the library. + */ + public static synchronized void loadLibrary(String libraryName) { + loadLibrary(getDefaultDestDir(), libraryName); + } + + /** + * Loads the native library specified by the libraryName argument. + * The libraryName argument must not contain any platform specific + * prefix, file extension or path. + * + * @param libraryName the name of the library. * @param exportSymbols export symbols of library so that it can be used by other libs. */ public static synchronized void loadLibrary(String libraryName, boolean exportSymbols) { + loadLibrary(getDefaultDestDir(), libraryName, exportSymbols); + } + + /** + * Loads the native library specified by the libraryName argument. + * The libraryName argument must not contain any platform specific + * prefix, file extension or path. + * + * @param destDir The destination dir the library to be extracted. + * @param libraryName the name of the library. + */ + public static synchronized void loadLibrary(String destDir, String libraryName) { + loadLibrary(destDir, libraryName, false); + } + + /** + * Loads the native library specified by the libraryName argument. + * The libraryName argument must not contain any platform specific + * prefix, file extension or path. + * + * @param destDir The destination dir the library to be extracted. + * @param libraryName the name of the library. + * @param exportSymbols export symbols of library so that it can be used by other libs. + */ + public static synchronized void loadLibrary(String destDir, String libraryName, + boolean exportSymbols) { if (!loadedLibs.contains(libraryName)) { LOGGER.debug("Loading native library {}.", libraryName); // Load native library. String fileName = System.mapLibraryName(libraryName); - final String sessionDir = RayConfig.getInstance().sessionDir; - final File file = BinaryFileUtil.getNativeFile(sessionDir, fileName); + final File file = BinaryFileUtil.getNativeFile(destDir, fileName); if (exportSymbols) { // Expose library symbols using RTLD_GLOBAL which may be depended by other shared @@ -50,4 +77,17 @@ public class JniUtils { } } + /** + * Cache the result so that multiple calls return the same dest dir. + */ + private static synchronized String getDefaultDestDir() { + if (defaultDestDir == null) { + try { + defaultDestDir = Files.createTempDirectory("native_libs").toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return defaultDestDir; + } } diff --git a/java/runtime/src/main/resources/ray.default.conf b/java/runtime/src/main/resources/ray.default.conf index c65768672..c222493b6 100644 --- a/java/runtime/src/main/resources/ray.default.conf +++ b/java/runtime/src/main/resources/ray.default.conf @@ -18,9 +18,6 @@ ray { // `CLUSTER`: Ray is running on one or more nodes, with multiple processes. run-mode: CLUSTER - // Available resources on this node, for example "CPU:4,GPU:0". - resources: "" - // Configuration items about job. job { // If worker.mode is DRIVER, specify the job id. @@ -56,40 +53,12 @@ ray { max-backup-files: 10 } - // Custom worker jvm parameters. - worker.jvm-parameters: [] - - // Custom `java.library.path` - // Note, do not use `dir1:dir2` format, put each dir as a list item. - library.path: [] - - // Custom classpath. - // Note, do not use `dir1:dir2` format, put each dir as a list item. - classpath = [] - // ---------------------- // Redis configurations // ---------------------- redis { - // If `redis.server` isn't provided, which port we should use to start redis server. - // If `head-port` is not provided, it will be generated randomly. - // head-port: 6379 - // Below passwords should be consistent with the one defined in python/ray/ray_constants.py. - // The password used to start the redis server on the head node. - head-password: "5241590000000000" // The password used to connect to the redis server. password: "5241590000000000" - // If `redis.server` isn't provided, how many Redis shards we should start in addition to the - // primary Redis shard. The ports of these shards will be `head-port + 1`, `head-port + 2`, etc. - shard-number: 1 - } - - // ---------------------------- - // Object store configurations - // ---------------------------- - object-store { - // Initial size of the object store. - size: 10 MB } // ---------------------------- @@ -97,12 +66,14 @@ ray { // ---------------------------- raylet { // See src/ray/ray_config_def.h for options. + // Below section takes effect only if Ray head is started by a driver. config { // TODO(zhuohan): enable this for java put_small_object_in_memory_store: false } } - // Whether we enable job manager to submit and manage job. - enable-job-manager: false + // Below args will be appended as parameters of the `ray start` command. + // It takes effect only if Ray head is started by a driver. + head-args: [] } diff --git a/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java b/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java index ee07bf8d0..6409bb037 100644 --- a/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java +++ b/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java @@ -2,6 +2,8 @@ package io.ray.runtime.config; import io.ray.runtime.generated.Common.WorkerType; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.testng.Assert; import org.testng.annotations.Test; @@ -11,37 +13,39 @@ public class RayConfigTest { @Test public void testCreateRayConfig() { + Map rayletConfig = new HashMap<>(); + rayletConfig.put("one", "1"); + rayletConfig.put("zero", "0"); + rayletConfig.put("positive-integer", "123"); + rayletConfig.put("negative-integer", "-123"); + rayletConfig.put("float", "-123.456"); + rayletConfig.put("true", "true"); + rayletConfig.put("false", "false"); + rayletConfig.put("string", "abc"); + try { System.setProperty("ray.job.code-search-path", "path/to/ray/job/resource/path"); + for (Map.Entry entry : rayletConfig.entrySet()) { + System.setProperty("ray.raylet.config." + entry.getKey(), entry.getValue()); + } RayConfig rayConfig = RayConfig.create(); Assert.assertEquals(WorkerType.DRIVER, rayConfig.workerMode); Assert.assertEquals(Collections.singletonList("path/to/ray/job/resource/path"), rayConfig.codeSearchPath); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("one"), 1); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("zero"), 0); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("positive-integer"), 123); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("negative-integer"), -123); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("float"), -123.456f); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("true"), true); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("false"), false); + Assert.assertEquals(rayConfig.rayletConfigParameters.get("string"), "abc"); } finally { // Unset system properties. System.clearProperty("ray.job.code-search-path"); - } - } - - @Test - public void testGenerateHeadPortRandomly() { - boolean isSame = true; - final int port1 = RayConfig.create().headRedisPort; - // If we the 2 ports are the same, let's retry. - // This is used to avoid any flaky chance. - for (int i = 0; i < NUM_RETRIES; ++i) { - final int port2 = RayConfig.create().headRedisPort; - if (port1 != port2) { - isSame = false; - break; + for (String key : rayletConfig.keySet()) { + System.clearProperty("ray.raylet.config." + key); } } - Assert.assertFalse(isSame); - } - - @Test - public void testSpecifyHeadPort() { - System.setProperty("ray.redis.head-port", "11111"); - Assert.assertEquals(RayConfig.create().headRedisPort, 11111); } } diff --git a/java/test.sh b/java/test.sh index 25fa94a1c..3352f5b33 100755 --- a/java/test.sh +++ b/java/test.sh @@ -39,13 +39,13 @@ echo "Build test jar." bazel build //java:all_tests_deploy.jar # Enable multi-worker feature in Java test -TEST_ARGS=(-Dray.raylet.config.num_workers_per_process_java=10 -Dray.job.num-java-workers-per-process=10) +TEST_ARGS=(-Dray.job.num-java-workers-per-process=10) echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, # TestNG will exit with code 2. And bazel treats it as test failure. -# bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --config=ci || cluster_exit_code=$? -ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml +# bazel test //java:all_tests --config=ci || cluster_exit_code=$? +run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar "${TEST_ARGS[@]}" org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml echo "Running tests under single-process mode." # bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --config=ci || single_exit_code=$? @@ -57,7 +57,7 @@ case "${OSTYPE}" in darwin*) ip=$(ipconfig getifaddr en0);; *) echo "Can't get ip address for ${OSTYPE}"; exit 1;; esac -RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --redis-password=123456 --code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar" +RAY_BACKEND_LOG_LEVEL=debug ray start --head --port=6379 --redis-password=123456 RAY_BACKEND_LOG_LEVEL=debug java -cp bazel-bin/java/all_tests_deploy.jar -Dray.address="$ip:6379"\ -Dray.redis.password='123456' -Dray.job.code-search-path="$PWD/bazel-bin/java/all_tests_deploy.jar" io.ray.test.MultiDriverTest ray stop diff --git a/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java b/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java deleted file mode 100644 index e4ffbf654..000000000 --- a/java/test/src/main/java/io/ray/test/BaseMultiLanguageTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package io.ray.test; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.gson.Gson; -import io.ray.api.Ray; -import io.ray.runtime.config.RayConfig; -import io.ray.runtime.util.NetworkUtil; -import java.io.File; -import java.lang.ProcessBuilder.Redirect; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -@Test(groups = {"cluster", "multiLanguage"}) -public abstract class BaseMultiLanguageTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(BaseMultiLanguageTest.class); - - private static final String PLASMA_STORE_SOCKET_NAME = "/tmp/ray/test/plasma_store_socket"; - private static final String RAYLET_SOCKET_NAME = "/tmp/ray/test/raylet_socket"; - - /** - * Execute an external command. - * - * @return Whether the command succeeded. - */ - private boolean executeCommand(List command, int waitTimeoutSeconds, - Map env) { - try { - LOGGER.info("Executing command: {}", String.join(" ", command)); - ProcessBuilder processBuilder = new ProcessBuilder(command).redirectOutput(Redirect.INHERIT) - .redirectError(Redirect.INHERIT); - for (Entry entry : env.entrySet()) { - processBuilder.environment().put(entry.getKey(), entry.getValue()); - } - Process process = processBuilder.start(); - process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); - return process.exitValue() == 0; - } catch (Exception e) { - throw new RuntimeException("Error executing command " + String.join(" ", command), e); - } - } - - @BeforeClass(alwaysRun = true, inheritGroups = false) - public void setUp() { - // Delete existing socket files. - for (String socket : ImmutableList.of(RAYLET_SOCKET_NAME, PLASMA_STORE_SOCKET_NAME)) { - File file = new File(socket); - if (file.exists()) { - file.delete(); - } - } - - String nodeManagerPort = String.valueOf(NetworkUtil.getUnusedPort()); - - // jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly. - // Since mvn test classes contains `test` in path and bazel test classes is located at a jar - // with `test` included in the name, we can check classpath `test` to filter out test classes. - List classpath = Stream.of(System.getProperty("java.class.path").split(":")) - .filter(s -> !s.contains(" ") && s.contains("test")) - .collect(Collectors.toList()); - // Start ray cluster. - List startCommand = Arrays.asList( - "ray", - "start", - "--head", - "--port=6379", - "--min-worker-port=0", - "--max-worker-port=0", - String.format("--plasma-store-socket-name=%s", PLASMA_STORE_SOCKET_NAME), - String.format("--raylet-socket-name=%s", RAYLET_SOCKET_NAME), - String.format("--node-manager-port=%s", nodeManagerPort), - "--load-code-from-local", - "--system-config=" + new Gson().toJson(RayConfig.create().rayletConfigParameters), - "--code-search-path=" + String.join(":", classpath) - ); - - if (!executeCommand(startCommand, 10, getRayStartEnv())) { - throw new RuntimeException("Couldn't start ray cluster."); - } - - // Connect to the cluster. - Assert.assertFalse(Ray.isInitialized()); - System.setProperty("ray.address", "127.0.0.1:6379"); - System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME); - System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME); - System.setProperty("ray.raylet.node-manager-port", nodeManagerPort); - Ray.init(); - } - - /** - * @return The environment variables needed for the `ray start` command. - */ - protected Map getRayStartEnv() { - return ImmutableMap.of(); - } - - @AfterClass(alwaysRun = true, inheritGroups = false) - public void tearDown() { - // Disconnect to the cluster. - Ray.shutdown(); - System.clearProperty("ray.address"); - System.clearProperty("ray.object-store.socket-name"); - System.clearProperty("ray.raylet.socket-name"); - System.clearProperty("ray.raylet.node-manager-port"); - - // Stop ray cluster. - final List stopCommand = ImmutableList.of( - "ray", - "stop" - ); - if (!executeCommand(stopCommand, 10, ImmutableMap.of())) { - throw new RuntimeException("Couldn't stop ray cluster"); - } - } -} diff --git a/java/test/src/main/java/io/ray/test/BaseTest.java b/java/test/src/main/java/io/ray/test/BaseTest.java index 07f2dbf4f..68698dbf9 100644 --- a/java/test/src/main/java/io/ray/test/BaseTest.java +++ b/java/test/src/main/java/io/ray/test/BaseTest.java @@ -1,46 +1,22 @@ package io.ray.test; -import com.google.common.collect.ImmutableList; import io.ray.api.Ray; -import java.io.File; import java.lang.reflect.Method; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; public class BaseTest { - private static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class); - - private List filesToDelete = ImmutableList.of(); - @BeforeMethod(alwaysRun = true) public void setUpBase(Method method) { Assert.assertFalse(Ray.isInitialized()); Ray.init(); - // These files need to be deleted after each test case. - filesToDelete = ImmutableList.of( - new File(Ray.getRuntimeContext().getRayletSocketName()), - new File(Ray.getRuntimeContext().getObjectStoreSocketName()), - // TODO(pcm): This is a workaround for the issue described - // in the PR description of https://github.com/ray-project/ray/pull/5450 - // and should be fixed properly. - new File("/tmp/ray/test/raylet_socket") - ); - // Make sure the files will be deleted even if the test doesn't exit gracefully. - filesToDelete.forEach(File::deleteOnExit); } @AfterMethod(alwaysRun = true) public void tearDownBase() { Ray.shutdown(); - - for (File file : filesToDelete) { - file.delete(); - } } } diff --git a/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java b/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java index 88e7f2d6c..5db99230f 100644 --- a/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java +++ b/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java @@ -1,7 +1,6 @@ package io.ray.test; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import io.ray.api.ActorHandle; import io.ray.api.ObjectRef; import io.ray.api.PyActorHandle; @@ -19,17 +18,19 @@ import java.io.InputStream; import java.math.BigInteger; import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.commons.io.FileUtils; import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -public class CrossLanguageInvocationTest extends BaseMultiLanguageTest { +@Test(groups = {"cluster"}) +public class CrossLanguageInvocationTest extends BaseTest { private static final String PYTHON_MODULE = "test_cross_language_invocation"; - @Override - protected Map getRayStartEnv() { + @BeforeClass + public void beforeClass() { // Delete and re-create the temp dir. File tempDir = new File( System.getProperty("java.io.tmpdir") + File.separator + "ray_cross_language_test"); @@ -48,7 +49,14 @@ public class CrossLanguageInvocationTest extends BaseMultiLanguageTest { throw new RuntimeException(e); } - return ImmutableMap.of("PYTHONPATH", tempDir.getAbsolutePath()); + System.setProperty("ray.job.code-search-path", + System.getProperty("java.class.path") + File.pathSeparator + + tempDir.getAbsolutePath()); + } + + @AfterClass + public void afterClass() { + System.clearProperty("ray.job.code-search-path"); } @Test diff --git a/java/test/src/main/java/io/ray/test/FailureTest.java b/java/test/src/main/java/io/ray/test/FailureTest.java index 6529b1beb..923875d5f 100644 --- a/java/test/src/main/java/io/ray/test/FailureTest.java +++ b/java/test/src/main/java/io/ray/test/FailureTest.java @@ -30,13 +30,13 @@ public class FailureTest extends BaseTest { // This is needed by `testGetThrowsQuicklyWhenFoundException`. // Set one worker per process. Otherwise, if `badFunc2` and `slowFunc` run in the same // process, `sleep` will delay `System.exit`. - oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java"); - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); + oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process"); + System.setProperty("ray.job.num-java-workers-per-process", "1"); } @AfterClass public void tearDown() { - System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess); + System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess); } public static int badFunc() { diff --git a/java/test/src/main/java/io/ray/test/GcsClientTest.java b/java/test/src/main/java/io/ray/test/GcsClientTest.java index d17e16a8b..45ed69d23 100644 --- a/java/test/src/main/java/io/ray/test/GcsClientTest.java +++ b/java/test/src/main/java/io/ray/test/GcsClientTest.java @@ -16,12 +16,12 @@ public class GcsClientTest extends BaseTest { @BeforeClass public void setUp() { - System.setProperty("ray.resources", "A:8"); + System.setProperty("ray.head-args.0", "--resources={\"A\":8}"); } @AfterClass public void tearDown() { - System.clearProperty("ray.resources"); + System.clearProperty("ray.head-args.0"); } public void testGetAllNodeInfo() { diff --git a/java/test/src/main/java/io/ray/test/GlobalGcTest.java b/java/test/src/main/java/io/ray/test/GlobalGcTest.java index a75da086e..f7acdf990 100644 --- a/java/test/src/main/java/io/ray/test/GlobalGcTest.java +++ b/java/test/src/main/java/io/ray/test/GlobalGcTest.java @@ -17,12 +17,12 @@ public class GlobalGcTest extends BaseTest { @BeforeClass public void setUp() { - System.setProperty("ray.object-store.size", "140 MB"); + System.setProperty("ray.head-args.0", "--object-store-memory=" + 140L * 1024 * 1024); } @AfterClass public void tearDown() { - System.clearProperty("ray.object-store.size"); + System.clearProperty("ray.head-args.0"); } public static class LargeObjectWithCyclicRef { diff --git a/java/test/src/main/java/io/ray/test/JobConfigTest.java b/java/test/src/main/java/io/ray/test/JobConfigTest.java index f7a7b79c3..d4725131e 100644 --- a/java/test/src/main/java/io/ray/test/JobConfigTest.java +++ b/java/test/src/main/java/io/ray/test/JobConfigTest.java @@ -14,7 +14,6 @@ public class JobConfigTest extends BaseTest { @BeforeClass public void setupJobConfig() { - System.setProperty("ray.raylet.config.enable_multi_tenancy", "true"); oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process"); System.setProperty("ray.job.num-java-workers-per-process", "3"); System.setProperty("ray.job.jvm-options.0", "-DX=999"); @@ -25,7 +24,6 @@ public class JobConfigTest extends BaseTest { @AfterClass public void tearDownJobConfig() { - System.clearProperty("ray.raylet.config.enable_multi_tenancy"); System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess); System.clearProperty("ray.job.jvm-options.0"); System.clearProperty("ray.job.jvm-options.1"); diff --git a/java/test/src/main/java/io/ray/test/KillActorTest.java b/java/test/src/main/java/io/ray/test/KillActorTest.java index cf88896d8..88b5b483b 100644 --- a/java/test/src/main/java/io/ray/test/KillActorTest.java +++ b/java/test/src/main/java/io/ray/test/KillActorTest.java @@ -18,13 +18,13 @@ public class KillActorTest extends BaseTest { @BeforeClass public void setUp() { - oldNumWorkersPerProcess = System.getProperty("ray.raylet.config.num_workers_per_process_java"); - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); + oldNumWorkersPerProcess = System.getProperty("ray.job.num-java-workers-per-process"); + System.setProperty("ray.job.num-java-workers-per-process", "1"); } @AfterClass public void tearDown() { - System.setProperty("ray.raylet.config.num_workers_per_process_java", oldNumWorkersPerProcess); + System.setProperty("ray.job.num-java-workers-per-process", oldNumWorkersPerProcess); } public static class HangActor { diff --git a/java/test/src/main/java/io/ray/test/MultiDriverTest.java b/java/test/src/main/java/io/ray/test/MultiDriverTest.java index f976ce18c..80d24d79d 100644 --- a/java/test/src/main/java/io/ray/test/MultiDriverTest.java +++ b/java/test/src/main/java/io/ray/test/MultiDriverTest.java @@ -15,8 +15,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @Test(groups = {"cluster"}) @@ -27,16 +25,6 @@ public class MultiDriverTest extends BaseTest { private static final int ACTOR_COUNT_PER_DRIVER = 10; private static final String PID_LIST_PREFIX = "PID: "; - @BeforeClass - public void setUpClass() { - System.setProperty("ray.raylet.config.enable_multi_tenancy", "true"); - } - - @AfterClass - public void tearDownClass() { - System.clearProperty("ray.raylet.config.enable_multi_tenancy"); - } - static int getPid() { return SystemUtil.pid(); } diff --git a/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java b/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java index 7ff3b53fe..0b244a841 100644 --- a/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java +++ b/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java @@ -5,7 +5,8 @@ import io.ray.api.Ray; import org.testng.Assert; import org.testng.annotations.Test; -public class MultiLanguageClusterTest extends BaseMultiLanguageTest { +@Test(groups = {"cluster"}) +public class MultiLanguageClusterTest extends BaseTest { public static String echo(String word) { return word; diff --git a/java/test/src/main/java/io/ray/test/PlacementGroupTest.java b/java/test/src/main/java/io/ray/test/PlacementGroupTest.java index 39fea16e9..547d3bae6 100644 --- a/java/test/src/main/java/io/ray/test/PlacementGroupTest.java +++ b/java/test/src/main/java/io/ray/test/PlacementGroupTest.java @@ -6,6 +6,7 @@ import io.ray.api.id.ActorId; import io.ray.api.placementgroup.PlacementGroup; import io.ray.api.placementgroup.PlacementGroupState; import io.ray.api.placementgroup.PlacementStrategy; +import io.ray.runtime.exception.RayException; import io.ray.runtime.placementgroup.PlacementGroupImpl; import java.util.List; import org.testng.Assert; @@ -31,8 +32,10 @@ public class PlacementGroupTest extends BaseTest { // This test just creates a placement group with one bundle. // It's not comprehensive to test all placement group test cases. public void testCreateAndCallActor() { - PlacementGroup placementGroup = PlacementGroupTestUtils.createSimpleGroup(); - Assert.assertEquals(((PlacementGroupImpl)placementGroup).getName(),"unnamed_group"); + PlacementGroupImpl placementGroup = (PlacementGroupImpl)PlacementGroupTestUtils + .createSimpleGroup(); + Assert.assertTrue(placementGroup.wait(10)); + Assert.assertEquals(placementGroup.getName(),"unnamed_group"); // Test creating an actor from a constructor. ActorHandle actor = Ray.actor(Counter::new, 1) @@ -40,7 +43,7 @@ public class PlacementGroupTest extends BaseTest { Assert.assertNotEquals(actor.getId(), ActorId.NIL); // Test calling an actor. - Assert.assertEquals(Integer.valueOf(1), actor.task(Counter::getValue).remote().get()); + Assert.assertEquals(actor.task(Counter::getValue).remote().get(), Integer.valueOf(1)); } @Test(groups = {"cluster"}) @@ -52,6 +55,8 @@ public class PlacementGroupTest extends BaseTest { PlacementGroupImpl secondPlacementGroup = (PlacementGroupImpl)PlacementGroupTestUtils .createNameSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK, 1.0, "second_placement_group"); + Assert.assertTrue(firstPlacementGroup.wait(10)); + Assert.assertTrue(secondPlacementGroup.wait(10)); PlacementGroupImpl firstPlacementGroupRes = (PlacementGroupImpl)Ray.getPlacementGroup((firstPlacementGroup).getId()); @@ -97,6 +102,15 @@ public class PlacementGroupTest extends BaseTest { PlacementGroupImpl removedPlacementGroup = (PlacementGroupImpl)Ray.getPlacementGroup((secondPlacementGroup).getId()); Assert.assertEquals(removedPlacementGroup.getState(), PlacementGroupState.REMOVED); + + // Wait for placement group after it is removed. + int exceptionCount = 0; + try { + removedPlacementGroup.wait(10); + } catch (RayException e) { + ++exceptionCount; + } + Assert.assertEquals(exceptionCount, 1); } public void testCheckBundleIndex() { @@ -108,14 +122,14 @@ public class PlacementGroupTest extends BaseTest { } catch (IllegalArgumentException e) { ++exceptionCount; } - Assert.assertEquals(1, exceptionCount); + Assert.assertEquals(exceptionCount, 1); try { Ray.actor(Counter::new, 1).setPlacementGroup(placementGroup, -1).remote(); } catch (IllegalArgumentException e) { ++exceptionCount; } - Assert.assertEquals(2, exceptionCount); + Assert.assertEquals(exceptionCount, 2); } @Test (expectedExceptions = { IllegalArgumentException.class }) diff --git a/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java index 3544b2dc4..7a6edbe95 100644 --- a/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java +++ b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java @@ -18,9 +18,6 @@ public class RayAlterSuiteListener implements IAlterSuiteListener { XmlGroups groups = new XmlGroups(); XmlRun run = new XmlRun(); run.onExclude(excludedGroup); - if (!"1".equals(System.getenv("ENABLE_MULTI_LANGUAGE_TESTS"))) { - run.onExclude("multiLanguage"); - } groups.setRun(run); suite.setGroups(groups); } diff --git a/java/test/src/main/java/io/ray/test/RaySerializerTest.java b/java/test/src/main/java/io/ray/test/RaySerializerTest.java index 6fc474a86..d9d2e335f 100644 --- a/java/test/src/main/java/io/ray/test/RaySerializerTest.java +++ b/java/test/src/main/java/io/ray/test/RaySerializerTest.java @@ -8,7 +8,8 @@ import io.ray.runtime.object.ObjectSerializer; import org.testng.Assert; import org.testng.annotations.Test; -public class RaySerializerTest extends BaseMultiLanguageTest { +@Test(groups = {"cluster"}) +public class RaySerializerTest extends BaseTest { @Test public void testSerializePyActor() { diff --git a/java/test/src/main/java/io/ray/test/RayletConfigTest.java b/java/test/src/main/java/io/ray/test/RayletConfigTest.java index 18d87d00f..81c7ce1d3 100644 --- a/java/test/src/main/java/io/ray/test/RayletConfigTest.java +++ b/java/test/src/main/java/io/ray/test/RayletConfigTest.java @@ -25,7 +25,9 @@ public class RayletConfigTest extends BaseTest { public static class TestActor { public String getConfigValue() { - return TestUtils.getRuntime().getRayConfig().rayletConfigParameters.get(RAY_CONFIG_KEY); + return TestUtils.getRuntime().getRayConfig() + .rayletConfigParameters.get(RAY_CONFIG_KEY) + .toString(); } } diff --git a/java/test/src/main/java/io/ray/test/RedisPasswordTest.java b/java/test/src/main/java/io/ray/test/RedisPasswordTest.java index 65d9f9bd1..c9317ed29 100644 --- a/java/test/src/main/java/io/ray/test/RedisPasswordTest.java +++ b/java/test/src/main/java/io/ray/test/RedisPasswordTest.java @@ -11,13 +11,11 @@ public class RedisPasswordTest extends BaseTest { @BeforeClass public void setUp() { - System.setProperty("ray.redis.head-password", "12345678"); System.setProperty("ray.redis.password", "12345678"); } @AfterClass public void tearDown() { - System.clearProperty("ray.redis.head-password"); System.clearProperty("ray.redis.password"); } diff --git a/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java b/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java index 374afbf8c..0d8e40856 100644 --- a/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java +++ b/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java @@ -27,12 +27,12 @@ import org.testng.annotations.Test; public class ReferenceCountingTest extends BaseTest { @BeforeClass public void setUp() { - System.setProperty("ray.object-store.size", "100 MB"); + System.setProperty("ray.head-args.0", "--object-store-memory=" + 100L * 1024 * 1024); } @AfterClass public void tearDown() { - System.clearProperty("ray.object-store.size"); + System.clearProperty("ray.head-args.0"); } /** diff --git a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java index 68ac20ed6..780f14087 100644 --- a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java +++ b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java @@ -20,12 +20,14 @@ public class ResourcesManagementTest extends BaseTest { @BeforeClass public void setUp() { - System.setProperty("ray.resources", "CPU:4,RES-A:4"); + System.setProperty("ray.head-args.0", "--num-cpus=4"); + System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}"); } @AfterClass public void tearDown() { - System.clearProperty("ray.resources"); + System.clearProperty("ray.head-args.0"); + System.clearProperty("ray.head-args.1"); } public static Integer echo(Integer number) { diff --git a/java/test/src/main/java/io/ray/test/RuntimeContextTest.java b/java/test/src/main/java/io/ray/test/RuntimeContextTest.java index 4f0385431..f8efbef04 100644 --- a/java/test/src/main/java/io/ray/test/RuntimeContextTest.java +++ b/java/test/src/main/java/io/ray/test/RuntimeContextTest.java @@ -14,8 +14,6 @@ import org.testng.annotations.Test; public class RuntimeContextTest extends BaseTest { private static JobId JOB_ID = getJobId(); - private static String RAYLET_SOCKET_NAME = "/tmp/ray/test/raylet_socket"; - private static String OBJECT_STORE_SOCKET_NAME = "/tmp/ray/test/object_store_socket"; private static JobId getJobId() { // Must be stable across different processes. @@ -27,23 +25,16 @@ public class RuntimeContextTest extends BaseTest { @BeforeClass public void setUp() { System.setProperty("ray.job.id", JOB_ID.toString()); - System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME); - System.setProperty("ray.object-store.socket-name", OBJECT_STORE_SOCKET_NAME); } @AfterClass public void tearDown() { System.clearProperty("ray.job.id"); - System.clearProperty("ray.raylet.socket-name"); - System.clearProperty("ray.object-store.socket-name"); } @Test public void testRuntimeContextInDriver() { Assert.assertEquals(JOB_ID, Ray.getRuntimeContext().getCurrentJobId()); - Assert.assertEquals(RAYLET_SOCKET_NAME, Ray.getRuntimeContext().getRayletSocketName()); - Assert.assertEquals(OBJECT_STORE_SOCKET_NAME, - Ray.getRuntimeContext().getObjectStoreSocketName()); } public static class RuntimeContextTester { @@ -51,9 +42,6 @@ public class RuntimeContextTest extends BaseTest { public String testRuntimeContext(ActorId actorId) { Assert.assertEquals(JOB_ID, Ray.getRuntimeContext().getCurrentJobId()); Assert.assertEquals(actorId, Ray.getRuntimeContext().getCurrentActorId()); - Assert.assertEquals(RAYLET_SOCKET_NAME, Ray.getRuntimeContext().getRayletSocketName()); - Assert.assertEquals(OBJECT_STORE_SOCKET_NAME, - Ray.getRuntimeContext().getObjectStoreSocketName()); return "ok"; } } diff --git a/python/ray/__init__.py b/python/ray/__init__.py index 23b8ac2a7..848ea5910 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -101,7 +101,7 @@ from ray import util # noqa: E402 # Replaced with the current commit when building the wheels. __commit__ = "{{RAY_COMMIT_SHA}}" -__version__ = "1.1.0.dev0" +__version__ = "1.2.0.dev0" __all__ = [ "__version__", diff --git a/python/ray/_private/services.py b/python/ray/_private/services.py index 9ddc8f3b7..62adfd072 100644 --- a/python/ray/_private/services.py +++ b/python/ray/_private/services.py @@ -136,7 +136,7 @@ def find_redis_address(address=None): # --redis_address=123.456.78.910 --node_ip_address=123.456.78.910 # --raylet_socket_name=... --store_socket_name=... --object_manager_port=0 # --min_worker_port=10000 --max_worker_port=10999 - # --node_manager_port=58578 --redis_port=6379 --num_initial_workers=8 + # --node_manager_port=58578 --redis_port=6379 # --maximum_startup_concurrency=8 # --static_resource_list=node:123.456.78.910,1.0,object_store_memory,66 # --config_list=plasma_store_as_thread,True @@ -279,7 +279,8 @@ def get_address_info_from_redis_helper(redis_address, def get_address_info_from_redis(redis_address, node_ip_address, num_retries=5, - redis_password=None): + redis_password=None, + no_warning=False): counter = 0 while True: try: @@ -290,10 +291,11 @@ def get_address_info_from_redis(redis_address, raise # Some of the information may not be in Redis yet, so wait a little # bit. - logger.warning( - "Some processes that the driver needs to connect to have " - "not registered with Redis, so retrying. Have you run " - "'ray start' on this node?") + if not no_warning: + logger.warning( + "Some processes that the driver needs to connect to have " + "not registered with Redis, so retrying. Have you run " + "'ray start' on this node?") time.sleep(1) counter += 1 @@ -1251,13 +1253,11 @@ def start_raylet(redis_address, stderr_file=None, config=None, java_worker_options=None, - load_code_from_local=False, huge_pages=False, fate_share=None, socket_to_use=None, head_node=False, - start_initial_python_workers_for_first_job=False, - code_search_path=None): + start_initial_python_workers_for_first_job=False): """Start a raylet, which is a combined local scheduler and object manager. Args: @@ -1294,9 +1294,6 @@ def start_raylet(redis_address, config (dict|None): Optional Raylet configuration that will override defaults in RayConfig. java_worker_options (list): The command options for Java worker. - code_search_path (list): Code search path for worker. code_search_path - is added to worker command in non-multi-tenancy mode and job_config - in multi-tenancy mode. Returns: ProcessInfo for the process that was started. """ @@ -1309,7 +1306,6 @@ def start_raylet(redis_address, raise ValueError("Cannot use valgrind and profiler at the same time.") assert resource_spec.resolved() - num_initial_workers = resource_spec.num_cpus static_resources = resource_spec.to_resource_dict() # Limit the number of workers that can be started in parallel by the @@ -1346,7 +1342,6 @@ def start_raylet(redis_address, raylet_name, redis_password, session_dir, - code_search_path, ) else: java_worker_command = [] @@ -1366,15 +1361,18 @@ def start_raylet(redis_address, # Create the command that the Raylet will use to start workers. start_worker_command = [ - sys.executable, worker_path, f"--node-ip-address={node_ip_address}", + sys.executable, + worker_path, + f"--node-ip-address={node_ip_address}", f"--node-manager-port={node_manager_port}", f"--object-store-name={plasma_store_name}", - f"--raylet-name={raylet_name}", f"--redis-address={redis_address}", - f"--config-list={config_str}", f"--temp-dir={temp_dir}", - f"--metrics-agent-port={metrics_agent_port}" + f"--raylet-name={raylet_name}", + f"--redis-address={redis_address}", + f"--config-list={config_str}", + f"--temp-dir={temp_dir}", + f"--metrics-agent-port={metrics_agent_port}", + "RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER", ] - if code_search_path: - start_worker_command.append(f"--code-search-path={code_search_path}") if redis_password: start_worker_command += [f"--redis-password={redis_password}"] @@ -1389,12 +1387,6 @@ def start_raylet(redis_address, if max_worker_port is None: max_worker_port = 0 - if code_search_path is not None and len(code_search_path) > 0: - load_code_from_local = True - - if load_code_from_local: - start_worker_command += ["--load-code-from-local"] - # Create agent command agent_command = [ sys.executable, @@ -1425,7 +1417,6 @@ def start_raylet(redis_address, f"--node_ip_address={node_ip_address}", f"--redis_address={gcs_ip_address}", f"--redis_port={gcs_port}", - f"--num_initial_workers={num_initial_workers}", f"--maximum_startup_concurrency={maximum_startup_concurrency}", f"--static_resource_list={resource_argument}", f"--config_list={config_str}", @@ -1485,8 +1476,7 @@ def get_ray_jars_dir(): def build_java_worker_command(java_worker_options, redis_address, node_manager_port, plasma_store_name, - raylet_name, redis_password, session_dir, - code_search_path): + raylet_name, redis_password, session_dir): """This method assembles the command used to start a Java worker. Args: @@ -1497,7 +1487,6 @@ def build_java_worker_command(java_worker_options, redis_address, raylet_name (str): The name of the raylet socket to create. redis_password (str): The password of connect to redis. session_dir (str): The path of this session. - code_search_path (list): Teh job code search path. Returns: The command string for starting Java worker. """ @@ -1518,7 +1507,6 @@ def build_java_worker_command(java_worker_options, redis_address, pairs.append(("ray.home", RAY_HOME)) pairs.append(("ray.logging.dir", os.path.join(session_dir, "logs"))) pairs.append(("ray.session-dir", session_dir)) - pairs.append(("ray.job.code-search-path", code_search_path)) command = ["java"] + ["-D{}={}".format(*pair) for pair in pairs] command += ["RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER"] diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 6feaab7de..8a4d2ce6c 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -336,6 +336,7 @@ cdef execute_task( const c_vector[shared_ptr[CRayObject]] &c_args, const c_vector[CObjectID] &c_arg_reference_ids, const c_vector[CObjectID] &c_return_ids, + const c_string debugger_breakpoint, c_vector[shared_ptr[CRayObject]] *returns): worker = ray.worker.global_worker @@ -351,6 +352,18 @@ cdef execute_task( # Automatically restrict the GPUs available to this task. ray.utils.set_cuda_visible_devices(ray.get_gpu_ids()) + # Helper method used to exit current asyncio actor. + # This is called when a KeyboardInterrupt is received by the main thread. + # Upon receiving a KeyboardInterrupt signal, Ray will exit the current + # worker. If the worker is processing normal tasks, Ray treat it as task + # cancellation from ray.cancel(object_ref). If the worker is an asyncio + # actor, Ray will exit the actor. + def exit_current_actor_if_asyncio(): + if core_worker.current_actor_is_asyncio(): + error = SystemExit(0) + error.is_ray_terminate = True + raise error + function_descriptor = CFunctionDescriptorToPython( ray_function.GetFunctionDescriptor()) @@ -457,9 +470,26 @@ cdef execute_task( task_exception = True try: with ray.worker._changeproctitle(title, next_title): + if debugger_breakpoint != b"": + ray.util.pdb.set_trace( + breakpoint_uuid=debugger_breakpoint) outputs = function_executor(*args, **kwargs) + next_breakpoint = ( + ray.worker.global_worker.debugger_breakpoint) + if next_breakpoint != b"": + # If this happens, the user typed "remote" and + # there were no more remote calls left in this + # task. In that case we just exit the debugger. + ray.experimental.internal_kv._internal_kv_put( + "RAY_PDB_{}".format(next_breakpoint), + "{\"exit_debugger\": true}") + ray.experimental.internal_kv._internal_kv_del( + "RAY_PDB_CONTINUE_{}".format(next_breakpoint) + ) + ray.worker.global_worker.debugger_breakpoint = b"" task_exception = False except KeyboardInterrupt as e: + exit_current_actor_if_asyncio() raise TaskCancelledError( core_worker.get_current_task_id()) if c_return_ids.size() == 1: @@ -467,6 +497,7 @@ cdef execute_task( # Check for a cancellation that was called when the function # was exiting and was raised after the except block. if not check_signals().ok(): + exit_current_actor_if_asyncio() task_exception = True raise TaskCancelledError( core_worker.get_current_task_id()) @@ -523,6 +554,7 @@ cdef CRayStatus task_execution_handler( const c_vector[shared_ptr[CRayObject]] &c_args, const c_vector[CObjectID] &c_arg_reference_ids, const c_vector[CObjectID] &c_return_ids, + const c_string debugger_breakpoint, c_vector[shared_ptr[CRayObject]] *returns) nogil: with gil: @@ -532,7 +564,7 @@ cdef CRayStatus task_execution_handler( # it does, that indicates that there was an internal error. execute_task(task_type, task_name, ray_function, c_resources, c_args, c_arg_reference_ids, c_return_ids, - returns) + debugger_breakpoint, returns) except Exception: traceback_str = traceback.format_exc() + ( "An unexpected internal error occurred while the worker " @@ -1041,6 +1073,7 @@ cdef class CoreWorker: PlacementGroupID placement_group_id, int64_t placement_group_bundle_index, c_bool placement_group_capture_child_tasks, + c_string debugger_breakpoint, override_environment_variables): cdef: unordered_map[c_string, double] c_resources @@ -1059,15 +1092,18 @@ cdef class CoreWorker: language.lang, function_descriptor.descriptor) prepare_args(self, language, args, &args_vector) - with nogil: - CCoreWorkerProcess.GetCoreWorker().SubmitTask( - ray_function, args_vector, CTaskOptions( - name, num_returns, c_resources, - c_override_environment_variables), - &return_ids, max_retries, - c_pair[CPlacementGroupID, int64_t]( - c_placement_group_id, placement_group_bundle_index), - placement_group_capture_child_tasks) + # NOTE(edoakes): releasing the GIL while calling this method causes + # segfaults. See relevant issue for details: + # https://github.com/ray-project/ray/pull/12803 + CCoreWorkerProcess.GetCoreWorker().SubmitTask( + ray_function, args_vector, CTaskOptions( + name, num_returns, c_resources, + c_override_environment_variables), + &return_ids, max_retries, + c_pair[CPlacementGroupID, int64_t]( + c_placement_group_id, placement_group_bundle_index), + placement_group_capture_child_tasks, + debugger_breakpoint) return VectorToObjectRefs(return_ids) @@ -1170,6 +1206,21 @@ cdef class CoreWorker: CCoreWorkerProcess.GetCoreWorker(). RemovePlacementGroup(c_placement_group_id)) + def wait_placement_group_ready(self, + PlacementGroupID placement_group_id, + int32_t timeout_seconds): + cdef CRayStatus status + cdef CPlacementGroupID cplacement_group_id = ( + CPlacementGroupID.FromBinary(placement_group_id.binary())) + cdef int ctimeout_seconds = timeout_seconds + with nogil: + status = CCoreWorkerProcess.GetCoreWorker() \ + .WaitPlacementGroupReady(cplacement_group_id, ctimeout_seconds) + if status.IsNotFound(): + raise Exception("Placement group {} does not exist.".format( + placement_group_id)) + return status.ok() + def submit_actor_task(self, Language language, ActorID actor_id, @@ -1193,12 +1244,14 @@ cdef class CoreWorker: language.lang, function_descriptor.descriptor) prepare_args(self, language, args, &args_vector) - with nogil: - CCoreWorkerProcess.GetCoreWorker().SubmitActorTask( - c_actor_id, - ray_function, - args_vector, CTaskOptions(name, num_returns, c_resources), - &return_ids) + # NOTE(edoakes): releasing the GIL while calling this method causes + # segfaults. See relevant issue for details: + # https://github.com/ray-project/ray/pull/12803 + CCoreWorkerProcess.GetCoreWorker().SubmitActorTask( + c_actor_id, + ray_function, + args_vector, CTaskOptions(name, num_returns, c_resources), + &return_ids) return VectorToObjectRefs(return_ids) @@ -1400,8 +1453,16 @@ cdef class CoreWorker: context = worker.get_serialization_context() serialized_object = context.serialize(output) data_sizes.push_back(serialized_object.total_bytes) - metadatas.push_back( - string_to_buffer(serialized_object.metadata)) + metadata = serialized_object.metadata + if ray.worker.global_worker.debugger_get_breakpoint: + breakpoint = ( + ray.worker.global_worker.debugger_get_breakpoint) + metadata += ( + b"," + ray_constants.OBJECT_METADATA_DEBUG_PREFIX + + breakpoint.encode()) + # Reset debugging context of this worker. + ray.worker.global_worker.debugger_get_breakpoint = b"" + metadatas.push_back(string_to_buffer(metadata)) serialized_objects.append(serialized_object) contained_ids.push_back( ObjectRefsToVector(serialized_object.contained_object_refs) diff --git a/python/ray/actor.py b/python/ray/actor.py index b8981ca3d..73255856d 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -1,6 +1,7 @@ import inspect import logging import weakref +import _thread import ray.ray_constants as ray_constants import ray._raylet @@ -1006,6 +1007,7 @@ def exit_actor(): """Intentionally exit the current actor. This function is used to disconnect an actor and exit the worker. + Any ``atexit`` handlers installed in the actor will be run. Raises: Exception: An exception is raised if this is a driver or this @@ -1018,6 +1020,14 @@ def exit_actor(): ray.disconnect() # Disconnect global state from GCS. ray.state.state.disconnect() + + # In asyncio actor mode, we can't raise SystemExit because it will just + # quit the asycnio event loop thread, not the main thread. Instead, we + # raise an interrupt signal to the main thread to tell it to exit. + if worker.core_worker.current_actor_is_asyncio(): + _thread.interrupt_main() + return + # Set a flag to indicate this is an intentional actor exit. This # reduces log verbosity. exit = SystemExit(0) diff --git a/python/ray/autoscaler/_private/autoscaler.py b/python/ray/autoscaler/_private/autoscaler.py index b918df7f3..2e55ed151 100644 --- a/python/ray/autoscaler/_private/autoscaler.py +++ b/python/ray/autoscaler/_private/autoscaler.py @@ -13,18 +13,19 @@ import collections from ray.experimental.internal_kv import _internal_kv_put, \ _internal_kv_initialized -from ray.autoscaler.tags import (TAG_RAY_LAUNCH_CONFIG, TAG_RAY_RUNTIME_CONFIG, - TAG_RAY_FILE_MOUNTS_CONTENTS, - TAG_RAY_NODE_STATUS, TAG_RAY_NODE_KIND, - TAG_RAY_USER_NODE_TYPE, STATUS_UP_TO_DATE, - NODE_KIND_WORKER, NODE_KIND_UNMANAGED) +from ray.autoscaler.tags import ( + TAG_RAY_LAUNCH_CONFIG, TAG_RAY_RUNTIME_CONFIG, + TAG_RAY_FILE_MOUNTS_CONTENTS, TAG_RAY_NODE_STATUS, TAG_RAY_NODE_KIND, + TAG_RAY_USER_NODE_TYPE, STATUS_UP_TO_DATE, NODE_KIND_WORKER, + NODE_KIND_UNMANAGED, NODE_KIND_HEAD) from ray.autoscaler._private.providers import _get_node_provider from ray.autoscaler._private.updater import NodeUpdaterThread from ray.autoscaler._private.node_launcher import NodeLauncher from ray.autoscaler._private.resource_demand_scheduler import \ - ResourceDemandScheduler, NodeType, NodeID + get_bin_pack_residual, ResourceDemandScheduler, NodeType, NodeID, NodeIP, \ + ResourceDict from ray.autoscaler._private.util import ConcurrentCounter, validate_config, \ - with_head_node_ip, hash_launch_conf, hash_runtime_conf, \ + with_head_node_ip, hash_launch_conf, hash_runtime_conf, add_prefix, \ DEBUG_AUTOSCALING_STATUS, DEBUG_AUTOSCALING_ERROR from ray.autoscaler._private.constants import \ AUTOSCALER_MAX_NUM_FAILURES, AUTOSCALER_MAX_LAUNCH_BATCH, \ @@ -47,7 +48,7 @@ class StandardAutoscaler: There are two ways to start an autoscaling cluster: manually by running `ray start --head --autoscaling-config=/path/to/config.yaml` on a instance that has permission to launch other instances, or you can also use - `ray create_or_update /path/to/config.yaml` from your laptop, which will + `ray up /path/to/config.yaml` from your laptop, which will configure the right AWS/Cloud roles automatically. StandardAutoscaler's `update` method is periodically called by `monitor.py` @@ -66,8 +67,11 @@ class StandardAutoscaler: max_concurrent_launches=AUTOSCALER_MAX_CONCURRENT_LAUNCHES, max_failures=AUTOSCALER_MAX_NUM_FAILURES, process_runner=subprocess, - update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S): + update_interval_s=AUTOSCALER_UPDATE_INTERVAL_S, + prefix_cluster_info=False): self.config_path = config_path + # Prefix each line of info string with cluster name if True + self.prefix_cluster_info = prefix_cluster_info # Keep this before self.reset (self.provider needs to be created # exactly once). self.provider = None @@ -164,27 +168,35 @@ class StandardAutoscaler: last_used = self.load_metrics.last_used_time_by_ip horizon = now - (60 * self.config["idle_timeout_minutes"]) - nodes_to_terminate = [] + nodes_to_terminate: Dict[NodeID, bool] = [] node_type_counts = collections.defaultdict(int) # Sort based on last used to make sure to keep min_workers that # were most recently used. Otherwise, _keep_min_workers_of_node_type # might keep a node that should be terminated. - for node_id in self._sort_based_on_last_used(nodes, last_used): + sorted_node_ids = self._sort_based_on_last_used(nodes, last_used) + # Don't terminate nodes needed by request_resources() + nodes_allowed_to_terminate: Dict[NodeID, bool] = {} + if self.resource_demand_vector: + nodes_allowed_to_terminate = self._get_nodes_allowed_to_terminate( + sorted_node_ids) + + for node_id in sorted_node_ids: # Make sure to not kill idle node types if the number of workers - # of that type is lower/equal to the min_workers of that type. - if self._keep_min_worker_of_node_type( - node_id, - node_type_counts) and self.launch_config_ok(node_id): + # of that type is lower/equal to the min_workers of that type + # or it is needed for request_resources(). + if (self._keep_min_worker_of_node_type(node_id, node_type_counts) + or not nodes_allowed_to_terminate.get( + node_id, True)) and self.launch_config_ok(node_id): continue node_ip = self.provider.internal_ip(node_id) if node_ip in last_used and last_used[node_ip] < horizon: logger.info("StandardAutoscaler: " - "{}: Terminating idle node".format(node_id)) + "{}: Terminating idle node.".format(node_id)) nodes_to_terminate.append(node_id) elif not self.launch_config_ok(node_id): logger.info("StandardAutoscaler: " - "{}: Terminating outdated node".format(node_id)) + "{}: Terminating outdated node.".format(node_id)) nodes_to_terminate.append(node_id) if nodes_to_terminate: @@ -198,7 +210,7 @@ class StandardAutoscaler: len(nodes_to_terminate)) > self.config["max_workers"] and nodes: to_terminate = nodes.pop() logger.info("StandardAutoscaler: " - "{}: Terminating unneeded node".format(to_terminate)) + "{}: Terminating unneeded node.".format(to_terminate)) nodes_to_terminate.append(to_terminate) if nodes_to_terminate: @@ -226,15 +238,23 @@ class StandardAutoscaler: if not updater.is_alive(): completed.append(node_id) if completed: + nodes_to_terminate: List[NodeID] = [] for node_id in completed: if self.updaters[node_id].exitcode == 0: self.num_successful_updates[node_id] += 1 + # Mark the node as active to prevent the node recovery + # logic immediately trying to restart Ray on the new node. + self.load_metrics.mark_active( + self.provider.internal_ip(node_id)) else: + logger.error(f"StandardAutoscaler: {node_id}: Terminating " + "failed to setup/initialize node.") + nodes_to_terminate.append(node_id) self.num_failed_updates[node_id] += 1 del self.updaters[node_id] - # Mark the node as active to prevent the node recovery logic - # immediately trying to restart Ray on the new node. - self.load_metrics.mark_active(self.provider.internal_ip(node_id)) + if nodes_to_terminate: + self.provider.terminate_nodes(nodes_to_terminate) + nodes = self.workers() self.log_info_string(nodes) @@ -266,14 +286,16 @@ class StandardAutoscaler: last_used: Dict[str, float]) -> List[NodeID]: """Sort the nodes based on the last time they were used. - The first item in the return list is the least recently used. + The first item in the return list is the most recently used. """ updated_last_used = copy.deepcopy(last_used) - now = time.time() + # Add the unconnected nodes as the least recently used (the end of + # list). This prioritizes connected nodes. + least_recently_used = -1 for node_id in nodes: node_ip = self.provider.internal_ip(node_id) if node_ip not in updated_last_used: - updated_last_used[node_ip] = now + updated_last_used[node_ip] = least_recently_used def last_time_used(node_id: NodeID): node_ip = self.provider.internal_ip(node_id) @@ -281,9 +303,86 @@ class StandardAutoscaler: return sorted(nodes, key=last_time_used, reverse=True) - def _keep_min_worker_of_node_type(self, node_id: NodeID, - node_type_counts: Dict[NodeType, int]): - """Returns if workers of node_type should be terminated. + def _get_nodes_allowed_to_terminate( + self, sorted_node_ids: List[NodeID]) -> Dict[NodeID, bool]: + # TODO(ameer): try merging this with resource_demand_scheduler + # code responsible for adding nodes for request_resources(). + """Returns the nodes allowed to terminate for request_resources(). + + Args: + sorted_node_ids: the node ids sorted based on last used (LRU last). + + Returns: + nodes_allowed_to_terminate: whether the node id is allowed to + terminate or not. + """ + nodes_allowed_to_terminate: Dict[NodeID, bool] = {} + head_node_resources: ResourceDict = copy.deepcopy( + self.available_node_types[self.config["head_node_type"]][ + "resources"]) + if not head_node_resources: + # Legacy yaml might include {} in the resources field. + # TODO(ameer): this is somewhat duplicated in + # resource_demand_scheduler.py. + head_id: List[NodeID] = self.provider.non_terminated_nodes({ + TAG_RAY_NODE_KIND: NODE_KIND_HEAD + }) + if head_id: + head_ip = self.provider.internal_ip(head_id[0]) + static_nodes: Dict[ + NodeIP, + ResourceDict] = \ + self.load_metrics.get_static_node_resources_by_ip() + head_node_resources = static_nodes[head_ip] + else: + head_node_resources = {} + + max_node_resources: List[ResourceDict] = [head_node_resources] + resource_demand_vector_worker_node_ids = [] + # Get max resources on all the non terminated nodes. + for node_id in sorted_node_ids: + tags = self.provider.node_tags(node_id) + if TAG_RAY_USER_NODE_TYPE in tags: + node_type = tags[TAG_RAY_USER_NODE_TYPE] + node_resources: ResourceDict = copy.deepcopy( + self.available_node_types[node_type]["resources"]) + if not node_resources: + # Legacy yaml might include {} in the resources field. + static_nodes: Dict[ + NodeIP, + ResourceDict] = \ + self.load_metrics.get_static_node_resources_by_ip() + node_ip = self.provider.internal_ip(node_id) + node_resources = static_nodes.get(node_ip, {}) + max_node_resources.append(node_resources) + resource_demand_vector_worker_node_ids.append(node_id) + # Since it is sorted based on last used, we "keep" nodes that are + # most recently used when we binpack. We assume get_bin_pack_residual + # is following the given order here. + used_resource_requests: List[ResourceDict] + _, used_resource_requests = \ + get_bin_pack_residual(max_node_resources, + self.resource_demand_vector) + # Remove the first entry (the head node). + max_node_resources.pop(0) + # Remove the first entry (the head node). + used_resource_requests.pop(0) + for i, node_id in enumerate(resource_demand_vector_worker_node_ids): + if used_resource_requests[i] == max_node_resources[i] \ + and max_node_resources[i]: + # No resources of the node were needed for request_resources(). + # max_node_resources[i] is an empty dict for legacy yamls + # before the node is connected. + nodes_allowed_to_terminate[node_id] = True + else: + nodes_allowed_to_terminate[node_id] = False + return nodes_allowed_to_terminate + + def _keep_min_worker_of_node_type( + self, node_id: NodeID, + node_type_counts: Dict[NodeType, int]) -> bool: + """Returns if workers of node_type can be terminated. + The worker cannot be terminated to respect min_workers constraint. Receives the counters of running nodes so far and determines if idle node_id should be terminated or not. It also updates the counters @@ -293,7 +392,7 @@ class StandardAutoscaler: node_type_counts(Dict[NodeType, int]): The non_terminated node types counted so far. Returns: - bool: if workers of node_types should be terminated or not. + bool: if workers of node_types can be terminated or not. """ tags = self.provider.node_tags(node_id) if TAG_RAY_USER_NODE_TYPE in tags: @@ -589,6 +688,8 @@ class StandardAutoscaler: self.load_metrics.get_resource_utilization()) if _internal_kv_initialized(): _internal_kv_put(DEBUG_AUTOSCALING_STATUS, tmp, overwrite=True) + if self.prefix_cluster_info: + tmp = add_prefix(tmp, self.config["cluster_name"]) logger.debug(tmp) def info_string(self, nodes): diff --git a/python/ray/autoscaler/_private/command_runner.py b/python/ray/autoscaler/_private/command_runner.py index 52ead65cd..075efa377 100644 --- a/python/ray/autoscaler/_private/command_runner.py +++ b/python/ray/autoscaler/_private/command_runner.py @@ -29,8 +29,6 @@ from ray.autoscaler._private.subprocess_output_util import ( from ray.autoscaler._private.cli_logger import cli_logger, cf from ray.util.debug import log_once -from ray.autoscaler._private.constants import RAY_HOME - logger = logging.getLogger(__name__) # How long to wait for a node to start, in seconds @@ -114,6 +112,7 @@ class KubernetesCommandRunner(CommandRunnerInterface): self.node_id = str(node_id) self.namespace = namespace self.kubectl = ["kubectl", "-n", self.namespace] + self._home_cached = None def run( self, @@ -195,7 +194,7 @@ class KubernetesCommandRunner(CommandRunnerInterface): logger.warning("'rsync_filter' detected but is currently " "unsupported for k8s.") if target.startswith("~"): - target = RAY_HOME + target[1:] + target = self._home + target[1:] try: flags = "-aqz" if is_rsync_silent() else "-avz" @@ -211,7 +210,7 @@ class KubernetesCommandRunner(CommandRunnerInterface): "rsync failed: '{}'. Falling back to 'kubectl cp'".format(e), UserWarning) if target.startswith("~"): - target = RAY_HOME + target[1:] + target = self._home + target[1:] self.process_runner.check_call(self.kubectl + [ "cp", source, "{}/{}:{}".format(self.namespace, self.node_id, @@ -219,8 +218,8 @@ class KubernetesCommandRunner(CommandRunnerInterface): ]) def run_rsync_down(self, source, target, options=None): - if target.startswith("~"): - target = RAY_HOME + target[1:] + if source.startswith("~"): + source = self._home + source[1:] try: flags = "-aqz" if is_rsync_silent() else "-avz" @@ -236,7 +235,7 @@ class KubernetesCommandRunner(CommandRunnerInterface): "rsync failed: '{}'. Falling back to 'kubectl cp'".format(e), UserWarning) if target.startswith("~"): - target = RAY_HOME + target[1:] + target = self._home + target[1:] self.process_runner.check_call(self.kubectl + [ "cp", "{}/{}:{}".format(self.namespace, self.node_id, source), @@ -244,8 +243,21 @@ class KubernetesCommandRunner(CommandRunnerInterface): ]) def remote_shell_command_str(self): - return "{} exec -it {} bash".format(" ".join(self.kubectl), - self.node_id) + return "{} exec -it {} -- bash".format(" ".join(self.kubectl), + self.node_id) + + @property + def _home(self): + # TODO (Dmitri): Think about how to use the node's HOME variable + # without making an extra kubectl exec call. + if self._home_cached is None: + cmd = self.kubectl + [ + "exec", "-it", self.node_id, "--", "printenv", "HOME" + ] + joined_cmd = " ".join(cmd) + raw_out = self.process_runner.check_output(joined_cmd, shell=True) + self._home_cached = raw_out.decode().strip("\n\r") + return self._home_cached class SSHOptions: diff --git a/python/ray/autoscaler/_private/kubernetes/__init__.py b/python/ray/autoscaler/_private/kubernetes/__init__.py index 300addc2b..e4754b76b 100644 --- a/python/ray/autoscaler/_private/kubernetes/__init__.py +++ b/python/ray/autoscaler/_private/kubernetes/__init__.py @@ -5,6 +5,7 @@ _configured = False _core_api = None _auth_api = None _extensions_beta_api = None +_custom_objects_api = None def _load_config(): @@ -45,4 +46,13 @@ def extensions_beta_api(): return _extensions_beta_api +def custom_objects_api(): + global _custom_objects_api + if _custom_objects_api is None: + _load_config() + _custom_objects_api = kubernetes.client.CustomObjectsApi() + + return _custom_objects_api + + log_prefix = "KubernetesNodeProvider: " diff --git a/python/ray/autoscaler/_private/kubernetes/config.py b/python/ray/autoscaler/_private/kubernetes/config.py index e6cc27a4c..20173119b 100644 --- a/python/ray/autoscaler/_private/kubernetes/config.py +++ b/python/ray/autoscaler/_private/kubernetes/config.py @@ -1,4 +1,6 @@ +import copy import logging +import math from kubernetes import client from kubernetes.client.rest import ApiException @@ -45,9 +47,10 @@ def not_provided_msg(resource_type): def bootstrap_kubernetes(config): if not config["provider"]["use_internal_ips"]: - return ValueError("Exposing external IP addresses for ray pods isn't " - "currently supported. Please set " - "'use_internal_ips' to false.") + return ValueError( + "Exposing external IP addresses for ray containers isn't " + "currently supported. Please set " + "'use_internal_ips' to false.") namespace = _configure_namespace(config["provider"]) _configure_autoscaler_service_account(namespace, config["provider"]) _configure_autoscaler_role(namespace, config["provider"]) @@ -56,6 +59,62 @@ def bootstrap_kubernetes(config): return config +def fillout_resources_kubernetes(config): + if "available_node_types" not in config: + return config["available_node_types"] + node_types = copy.deepcopy(config["available_node_types"]) + for node_type in node_types: + container_data = node_types[node_type]["node_config"]["spec"][ + "containers"][0] + autodetected_resources = get_autodetected_resources(container_data) + if "resources" not in config["available_node_types"][node_type]: + config["available_node_types"][node_type]["resources"] = {} + config["available_node_types"][node_type]["resources"].update( + autodetected_resources) + logger.debug( + "Updating the resources of node type {} to include {}.".format( + node_type, autodetected_resources)) + return config + + +def get_autodetected_resources(container_data): + container_resources = container_data.get("resources", None) + if container_resources is None: + return {"CPU": 0, "GPU": 0} + + node_type_resources = { + resource_name.upper(): get_resource(container_resources, resource_name) + for resource_name in ["cpu", "gpu"] + } + + return node_type_resources + + +def get_resource(container_resources, resource_name): + request = _get_resource( + container_resources, resource_name, field_name="requests") + limit = _get_resource( + container_resources, resource_name, field_name="limits") + resource = min(request, limit) + return 0 if resource == float("inf") else int(resource) + + +def _get_resource(container_resources, resource_name, field_name): + if (field_name in container_resources + and resource_name in container_resources[field_name]): + return _parse_resource(container_resources[field_name][resource_name]) + else: + return float("inf") + + +def _parse_resource(resource): + resource_str = str(resource) + if resource_str[-1] == "m": + return math.ceil(int(resource_str[:-1]) / 1000) + else: + return int(resource_str) + + def _configure_namespace(provider_config): namespace_field = "namespace" if namespace_field not in provider_config: diff --git a/python/ray/autoscaler/_private/kubernetes/node_provider.py b/python/ray/autoscaler/_private/kubernetes/node_provider.py index 997a28718..1fd3dc286 100644 --- a/python/ray/autoscaler/_private/kubernetes/node_provider.py +++ b/python/ray/autoscaler/_private/kubernetes/node_provider.py @@ -6,7 +6,8 @@ from kubernetes.client.rest import ApiException from ray.autoscaler._private.command_runner import KubernetesCommandRunner from ray.autoscaler._private.kubernetes import core_api, log_prefix, \ extensions_beta_api -from ray.autoscaler._private.kubernetes.config import bootstrap_kubernetes +from ray.autoscaler._private.kubernetes.config import bootstrap_kubernetes, \ + fillout_resources_kubernetes from ray.autoscaler.node_provider import NodeProvider from ray.autoscaler.tags import TAG_RAY_CLUSTER_NAME @@ -177,6 +178,11 @@ class KubernetesNodeProvider(NodeProvider): def bootstrap_config(cluster_config): return bootstrap_kubernetes(cluster_config) + @staticmethod + def fillout_available_node_types_resources(cluster_config): + """Fills out missing "resources" field for available_node_types.""" + return fillout_resources_kubernetes(cluster_config) + def _add_service_name_to_service_port(spec, svc_name): """Goes recursively through the ingress manifest and adds the diff --git a/python/ray/autoscaler/_private/load_metrics.py b/python/ray/autoscaler/_private/load_metrics.py index d5f3b73e9..1fadeae3b 100644 --- a/python/ray/autoscaler/_private/load_metrics.py +++ b/python/ray/autoscaler/_private/load_metrics.py @@ -82,12 +82,14 @@ class LoadMetrics: def prune(mapping): unwanted = set(mapping) - active_ips for unwanted_key in unwanted: - logger.info("LoadMetrics: " - "Removed mapping: {} - {}".format( - unwanted_key, mapping[unwanted_key])) + # TODO (Alex): Change this back to info after #12138. + logger.debug("LoadMetrics: " + "Removed mapping: {} - {}".format( + unwanted_key, mapping[unwanted_key])) del mapping[unwanted_key] if unwanted: - logger.info( + # TODO (Alex): Change this back to info after #12138. + logger.debug( "LoadMetrics: " "Removed {} stale ip mappings: {} not in {}".format( len(unwanted), unwanted, active_ips)) diff --git a/python/ray/autoscaler/_private/resource_demand_scheduler.py b/python/ray/autoscaler/_private/resource_demand_scheduler.py index e75364e8c..6bbae1762 100644 --- a/python/ray/autoscaler/_private/resource_demand_scheduler.py +++ b/python/ray/autoscaler/_private/resource_demand_scheduler.py @@ -135,24 +135,6 @@ class ResourceDemandScheduler: this set of resources. This differs from resources_demands in that we don't take into account existing usage. """ - - # If the user is using request_resources() API, calculate the remaining - # delta resources required to meet their requested cluster size. - if ensure_min_cluster_size is not None: - used_resources = [] - for ip, max_res in max_resources_by_ip.items(): - res = copy.deepcopy(max_res) - _inplace_subtract(res, unused_resources_by_ip.get(ip, {})) - used_resources.append(res) - # Example: user requests 1000 CPUs, but the cluster is currently - # 500 CPUs in size with 250 used. Then, the delta is 750 CPUs that - # we need to fit to get the cluster to scale to 1000. - resource_requests, _ = get_bin_pack_residual( - used_resources, ensure_min_cluster_size) - resource_demands += resource_requests - else: - resource_requests = [] - if self.is_legacy_yaml(): # When using legacy yaml files we need to infer the head & worker # node resources from the static node resources from LoadMetrics. @@ -166,9 +148,12 @@ class ResourceDemandScheduler: logger.info("Cluster resources: {}".format(node_resources)) logger.info("Node counts: {}".format(node_type_counts)) # Step 2: add nodes to add to satisfy min_workers for each type - node_resources, node_type_counts, min_workers_nodes_to_add = \ + (node_resources, + node_type_counts, + adjusted_min_workers) = \ _add_min_workers_nodes( - node_resources, node_type_counts, self.node_types) + node_resources, node_type_counts, self.node_types, + self.max_workers, ensure_min_cluster_size) # Step 3: add nodes for strict spread groups logger.info(f"Placement group demands: {pending_placement_groups}") @@ -180,8 +165,16 @@ class ResourceDemandScheduler: not self.node_types[NODE_TYPE_LEGACY_WORKER]["resources"]: # Need to launch worker nodes to later infer their # resources. + # We add request_resources() demands here to make sure we launch + # a single worker sometimes even if min_workers = 0 and resource + # demands is empty. + if ensure_min_cluster_size: + request_resources_demands = ensure_min_cluster_size + else: + request_resources_demands = [] return self._legacy_worker_node_to_launch( - nodes, launching_nodes, node_resources, resource_demands) + nodes, launching_nodes, node_resources, + resource_demands + request_resources_demands) placement_group_nodes_to_add, node_resources, node_type_counts = \ self.reserve_and_allocate_spread( strict_spreads, node_resources, node_type_counts) @@ -194,20 +187,15 @@ class ResourceDemandScheduler: logger.info("Unfulfilled demands: {}".format(unfulfilled)) # Add 1 to account for the head node. max_to_add = self.max_workers + 1 - sum(node_type_counts.values()) - if resource_requests: - nodes_to_add_based_on_requests = get_nodes_for( - self.node_types, node_type_counts, max_to_add, - resource_requests) - else: - nodes_to_add_based_on_requests = {} nodes_to_add_based_on_demand = get_nodes_for( self.node_types, node_type_counts, max_to_add, unfulfilled) # Merge nodes to add based on demand and nodes to add based on # min_workers constraint. We add them because nodes to add based on # demand was calculated after the min_workers constraint was respected. total_nodes_to_add = {} + for node_type in self.node_types: - nodes_to_add = (min_workers_nodes_to_add.get( + nodes_to_add = (adjusted_min_workers.get( node_type, 0) + placement_group_nodes_to_add.get(node_type, 0) + nodes_to_add_based_on_demand.get(node_type, 0)) if nodes_to_add > 0: @@ -216,7 +204,7 @@ class ResourceDemandScheduler: # Limit the number of concurrent launches total_nodes_to_add = self._get_concurrent_resource_demand_to_launch( total_nodes_to_add, unused_resources_by_ip.keys(), nodes, - launching_nodes, nodes_to_add_based_on_requests) + launching_nodes, adjusted_min_workers) logger.info("Node requests: {}".format(total_nodes_to_add)) return total_nodes_to_add @@ -294,7 +282,7 @@ class ResourceDemandScheduler: connected_nodes: List[NodeIP], non_terminated_nodes: List[NodeID], pending_launches_nodes: Dict[NodeType, int], - nodes_to_add_based_on_requests: Dict[NodeType, int], + adjusted_min_workers: Dict[NodeType, int], ) -> Dict[NodeType, int]: """Updates the max concurrent resources to launch for each node type. @@ -314,9 +302,10 @@ class ResourceDemandScheduler: connected_nodes: Running nodes (from LoadMetrics). non_terminated_nodes: Non terminated nodes (pending/running). pending_launches_nodes: Nodes that are in the launch queue. - nodes_to_add_based_on_requests: Nodes to launch to satisfy - request_resources(). This overrides the launch limits since the - user is hinting to immediately scale up to this size. + adjusted_min_workers: Nodes to launch to satisfy + min_workers and request_resources(). This overrides the launch + limits since the user is hinting to immediately scale up to + this size. Returns: Dict[NodeType, int]: Maximum number of nodes to launch for each node type. @@ -338,13 +327,9 @@ class ResourceDemandScheduler: upper_bound = max( max_allowed_pending_nodes - total_pending_nodes, - # Allow more nodes if this is to respect min_workers. - self.node_types[node_type].get("min_workers", 0) - - total_pending_nodes - running_nodes[node_type], - - # Allow more nodes from request_resources API. - nodes_to_add_based_on_requests.get(node_type, - 0) - total_pending_nodes) + # Allow more nodes if this is to respect min_workers or + # request_resources(). + adjusted_min_workers.get(node_type, 0)) if upper_bound > 0: updated_nodes_to_launch[node_type] = min( @@ -504,21 +489,26 @@ def _node_type_counts_to_node_resources( def _add_min_workers_nodes( node_resources: List[ResourceDict], node_type_counts: Dict[NodeType, int], - node_types: Dict[NodeType, NodeTypeConfigDict], + node_types: Dict[NodeType, NodeTypeConfigDict], max_workers: int, + ensure_min_cluster_size: List[ResourceDict] ) -> (List[ResourceDict], Dict[NodeType, int], Dict[NodeType, int]): - """Updates resource demands to respect the min_workers constraint. + """Updates resource demands to respect the min_workers and + request_resources() constraints. Args: node_resources: Resources of exisiting nodes already launched/pending. node_type_counts: Counts of existing nodes already launched/pending. node_types: Node types config. + max_workers: global max_workers constaint. + ensure_min_cluster_size: resource demands from request_resources(). Returns: node_resources: The updated node resources after adding min_workers - constraint per node type. + and request_resources() constraints per node type. node_type_counts: The updated node counts after adding min_workers - constraint per node type. - total_nodes_to_add: The nodes to add to respect min_workers constraint. + and request_resources() constraints per node type. + total_nodes_to_add_dict: The nodes to add to respect min_workers and + request_resources() constraints. """ total_nodes_to_add_dict = {} for node_type, config in node_types.items(): @@ -528,10 +518,41 @@ def _add_min_workers_nodes( if existing < target: total_nodes_to_add_dict[node_type] = target - existing node_type_counts[node_type] = target - available = copy.deepcopy(node_types[node_type]["resources"]) - node_resources.extend( - [available] * total_nodes_to_add_dict[node_type]) + node_resources.extend([ + copy.deepcopy(node_types[node_type]["resources"]) + for _ in range(total_nodes_to_add_dict[node_type]) + ]) + if ensure_min_cluster_size: + max_to_add = max_workers + 1 - sum(node_type_counts.values()) + max_node_resources = [] + # Fit request_resources() on all the resources as if they are idle. + for node_type in node_type_counts: + max_node_resources.extend([ + copy.deepcopy(node_types[node_type]["resources"]) + for _ in range(node_type_counts[node_type]) + ]) + # Get the unfulfilled to ensure min cluster size. + resource_requests_unfulfilled, _ = get_bin_pack_residual( + max_node_resources, ensure_min_cluster_size) + # Get the nodes to meet the unfulfilled. + nodes_to_add_request_resources = get_nodes_for( + node_types, node_type_counts, max_to_add, + resource_requests_unfulfilled) + # Update the resources, counts and total nodes to add. + for node_type in nodes_to_add_request_resources: + nodes_to_add = nodes_to_add_request_resources.get(node_type, 0) + if nodes_to_add > 0: + node_type_counts[ + node_type] = nodes_to_add + node_type_counts.get( + node_type, 0) + node_resources.extend([ + copy.deepcopy(node_types[node_type]["resources"]) + for _ in range(nodes_to_add) + ]) + total_nodes_to_add_dict[ + node_type] = nodes_to_add + total_nodes_to_add_dict.get( + node_type, 0) return node_resources, node_type_counts, total_nodes_to_add_dict @@ -623,7 +644,8 @@ def _utilization_score(node_resources: ResourceDict, def get_bin_pack_residual(node_resources: List[ResourceDict], resource_demands: List[ResourceDict], - strict_spread: bool = False) -> List[ResourceDict]: + strict_spread: bool = False + ) -> (List[ResourceDict], List[ResourceDict]): """Return a subset of resource_demands that cannot fit in the cluster. TODO(ekl): this currently does not guarantee the resources will be packed @@ -638,7 +660,7 @@ def get_bin_pack_residual(node_resources: List[ResourceDict], placed on a different entry in `node_resources`. Returns: - List[ResourceDict] the residual list resources that do not fit. + List[ResourceDict]: the residual list resources that do not fit. List[ResourceDict]: The updated node_resources after the method. """ diff --git a/python/ray/autoscaler/_private/updater.py b/python/ray/autoscaler/_private/updater.py index 0fbf84d94..8616fbe84 100644 --- a/python/ray/autoscaler/_private/updater.py +++ b/python/ray/autoscaler/_private/updater.py @@ -256,8 +256,16 @@ class NodeUpdater: retry_str = "(" + str(e) + ")" if hasattr(e, "cmd"): + if isinstance(e.cmd, str): + cmd_ = e.cmd + elif isinstance(e.cmd, list): + cmd_ = " ".join(e.cmd) + else: + logger.debug(f"e.cmd type ({type(e.cmd)}) not " + "list or str.") + cmd_ = str(e.cmd) retry_str = "(Exit Status {}): {}".format( - e.returncode, " ".join(e.cmd)) + e.returncode, cmd_) cli_logger.print( "SSH still not available {}, " diff --git a/python/ray/autoscaler/_private/util.py b/python/ray/autoscaler/_private/util.py index b500e846a..b4066df95 100644 --- a/python/ray/autoscaler/_private/util.py +++ b/python/ray/autoscaler/_private/util.py @@ -244,3 +244,14 @@ def hash_runtime_conf(file_mounts, file_mounts_contents_hash = None return (_hash_cache[conf_str], file_mounts_contents_hash) + + +def add_prefix(info_string, prefix): + """Prefixes each line of info_string, except the first, by prefix.""" + lines = info_string.split("\n") + prefixed_lines = [lines[0]] + for line in lines[1:]: + prefixed_line = ":".join([prefix, line]) + prefixed_lines.append(prefixed_line) + prefixed_info_string = "\n".join(prefixed_lines) + return prefixed_info_string diff --git a/python/ray/autoscaler/aws/defaults.yaml b/python/ray/autoscaler/aws/defaults.yaml index b85df640c..da2b5e089 100644 --- a/python/ray/autoscaler/aws/defaults.yaml +++ b/python/ray/autoscaler/aws/defaults.yaml @@ -112,7 +112,7 @@ setup_commands: # has your Ray repo pre-cloned. Then, you can replace the pip installs # below with a git checkout (and possibly a recompile). - echo 'export PATH="$HOME/anaconda3/envs/tensorflow_p36/bin:$PATH"' >> ~/.bashrc - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl # Consider uncommenting these if you also want to run apt-get commands during setup # - sudo pkill -9 apt-get || true # - sudo pkill -9 dpkg || true diff --git a/python/ray/autoscaler/aws/example-full.yaml b/python/ray/autoscaler/aws/example-full.yaml index f47a7523f..ffefc835f 100644 --- a/python/ray/autoscaler/aws/example-full.yaml +++ b/python/ray/autoscaler/aws/example-full.yaml @@ -19,7 +19,8 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + image: "rayproject/ray-ml:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_container" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -27,10 +28,10 @@ docker: run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" # Allow Ray to automatically detect GPUs - # worker_image: "rayproject/ray:latest-cpu" + # worker_image: "rayproject/ray-ml:latest-cpu" # worker_run_options: [] # If a node is idle for this many minutes, it will be removed. @@ -128,7 +129,7 @@ setup_commands: [] # has your Ray repo pre-cloned. Then, you can replace the pip installs # below with a git checkout (and possibly a recompile). # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest) - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: [] diff --git a/python/ray/autoscaler/aws/example-gpu-docker.yaml b/python/ray/autoscaler/aws/example-gpu-docker.yaml index 6916feb9c..4b5b052ff 100644 --- a/python/ray/autoscaler/aws/example-gpu-docker.yaml +++ b/python/ray/autoscaler/aws/example-gpu-docker.yaml @@ -19,13 +19,14 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" + image: "rayproject/ray-ml:latest-gpu" + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_nvidia_docker" # e.g. ray_docker # # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" - # worker_image: "rayproject/ray:latest" + # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 @@ -90,8 +91,8 @@ file_mounts: { # List of shell commands to run to set up nodes. # NOTE: rayproject/ray:latest has ray latest bundled setup_commands: [] - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: diff --git a/python/ray/autoscaler/aws/example-java.yaml b/python/ray/autoscaler/aws/example-java.yaml index e9bd11ad3..3d27807b2 100644 --- a/python/ray/autoscaler/aws/example-java.yaml +++ b/python/ray/autoscaler/aws/example-java.yaml @@ -2,7 +2,7 @@ cluster_name: java # The minimum number of workers nodes to launch in addition to the head # node. This number should be >= 0. -min_workers: 1 +min_workers: 1 # The maximum number of workers nodes to launch in addition to the head # node. This takes precedence over min_workers. max_workers: 1 @@ -72,10 +72,10 @@ worker_setup_commands: [] # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - ray stop - - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --code-search-path=~/ray-word-count/target + - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - ray stop - - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --code-search-path=ray-word-count/target + - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 -# To run the program, run `ray exec java.yaml "java -jar ray-word-count/target/ray-word-count-1.0-SNAPSHOT-jar-with-dependencies.jar"` +# To run the program, run `ray exec java.yaml "java -jar ray-word-count/target/ray-word-count-1.0-SNAPSHOT-jar-with-dependencies.jar -Dray.job.code-search-path=ray-word-count/target"` diff --git a/python/ray/autoscaler/aws/example-ml.yaml b/python/ray/autoscaler/aws/example-ml.yaml index 8da0baff6..972dc0082 100644 --- a/python/ray/autoscaler/aws/example-ml.yaml +++ b/python/ray/autoscaler/aws/example-ml.yaml @@ -24,7 +24,7 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "" # e.g., rayproject/ray:latest + image: "" # e.g., rayproject/ray-ml:latest container_name: "" # e.g. ray_docker # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -32,9 +32,9 @@ docker: run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" - # worker_image: "rayproject/ray:latest" + # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 @@ -120,7 +120,7 @@ setup_commands: # has your Ray repo pre-cloned. Then, you can replace the pip installs # below with a git checkout (and possibly a recompile). - source activate pytorch_p36 && pip install -U ray - - source activate pytorch_p36 && pip install -U ray[rllib] ray[tune] ray[debug] + - source activate pytorch_p36 && pip install -U ray[rllib] ray[tune] ray # Consider uncommenting these if you also want to run apt-get commands during setup # - sudo pkill -9 apt-get || true # - sudo pkill -9 dpkg || true diff --git a/python/ray/autoscaler/azure/defaults.yaml b/python/ray/autoscaler/azure/defaults.yaml index 4f2acdb3d..00322e5f9 100644 --- a/python/ray/autoscaler/azure/defaults.yaml +++ b/python/ray/autoscaler/azure/defaults.yaml @@ -112,7 +112,7 @@ setup_commands: - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc # - echo 'conda activate py37_pytorch' >> ~/.bashrc - echo 'conda activate py37_tensorflow' >> ~/.bashrc - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Consider uncommenting these if you also want to run apt-get commands during setup # - sudo pkill -9 apt-get || true # - sudo pkill -9 dpkg || true diff --git a/python/ray/autoscaler/azure/example-full.yaml b/python/ray/autoscaler/azure/example-full.yaml index 07fa6e495..9fad4e040 100644 --- a/python/ray/autoscaler/azure/example-full.yaml +++ b/python/ray/autoscaler/azure/example-full.yaml @@ -19,7 +19,8 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + image: "rayproject/ray-ml:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_container" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -27,10 +28,10 @@ docker: run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" # Allow Ray to automatically detect GPUs - # worker_image: "rayproject/ray:latest-cpu" + # worker_image: "rayproject/ray-ml:latest-cpu" # worker_run_options: [] # If a node is idle for this many minutes, it will be removed. @@ -128,7 +129,7 @@ setup_commands: # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest) - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc - echo 'conda activate py37_tensorflow' >> ~/.bashrc - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: diff --git a/python/ray/autoscaler/azure/example-gpu-docker.yaml b/python/ray/autoscaler/azure/example-gpu-docker.yaml index 11dcece40..b8173a19b 100644 --- a/python/ray/autoscaler/azure/example-gpu-docker.yaml +++ b/python/ray/autoscaler/azure/example-gpu-docker.yaml @@ -19,13 +19,14 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" + image: "rayproject/ray-ml:latest-gpu" + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_nvidia_docker" # e.g. ray_docker # # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" - # worker_image: "rayproject/ray:latest" + # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 @@ -65,9 +66,9 @@ file_mounts: { } # List of shell commands to run to set up nodes. -# NOTE: rayproject/ray:latest has ray latest bundled +# NOTE: rayproject/ray-ml:latest has ray latest bundled setup_commands: [] -# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl +# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: diff --git a/python/ray/autoscaler/azure/example-gpu.yaml b/python/ray/autoscaler/azure/example-gpu.yaml index 52d533b59..adbe18b2b 100644 --- a/python/ray/autoscaler/azure/example-gpu.yaml +++ b/python/ray/autoscaler/azure/example-gpu.yaml @@ -19,7 +19,8 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" + image: "rayproject/ray-ml:latest-gpu" + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_docker" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -27,9 +28,9 @@ docker: run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" - # worker_image: "rayproject/ray:latest" + # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 @@ -97,7 +98,7 @@ setup_commands: - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc # - echo 'conda activate py37_pytorch' >> ~/.bashrc - echo 'conda activate py37_tensorflow' >> ~/.bashrc - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Consider uncommenting these if you also want to run apt-get commands during setup # - sudo pkill -9 apt-get || true # - sudo pkill -9 dpkg || true diff --git a/python/ray/autoscaler/gcp/defaults.yaml b/python/ray/autoscaler/gcp/defaults.yaml index 667fa8f93..988ff93df 100644 --- a/python/ray/autoscaler/gcp/defaults.yaml +++ b/python/ray/autoscaler/gcp/defaults.yaml @@ -130,7 +130,7 @@ setup_commands: && echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.profile # Install ray - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. diff --git a/python/ray/autoscaler/gcp/example-full.yaml b/python/ray/autoscaler/gcp/example-full.yaml index 2f66d1dd3..d2e23efe8 100644 --- a/python/ray/autoscaler/gcp/example-full.yaml +++ b/python/ray/autoscaler/gcp/example-full.yaml @@ -19,7 +19,8 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + image: "rayproject/ray-ml:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_container" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -27,10 +28,10 @@ docker: run_options: [] # Extra options to pass into "docker run" # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" # Allow Ray to automatically detect GPUs - # worker_image: "rayproject/ray:latest-cpu" + # worker_image: "rayproject/ray-ml:latest-cpu" # worker_run_options: [] # If a node is idle for this many minutes, it will be removed. @@ -136,7 +137,7 @@ setup_commands: [] # has your Ray repo pre-cloned. Then, you can replace the pip installs # below with a git checkout (and possibly a recompile). # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest) - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. diff --git a/python/ray/autoscaler/gcp/example-gpu-docker.yaml b/python/ray/autoscaler/gcp/example-gpu-docker.yaml index 6552a8c89..12663eb55 100644 --- a/python/ray/autoscaler/gcp/example-gpu-docker.yaml +++ b/python/ray/autoscaler/gcp/example-gpu-docker.yaml @@ -19,14 +19,15 @@ upscaling_speed: 1.0 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. docker: - image: "rayproject/ray:latest-gpu" + image: "rayproject/ray-ml:latest-gpu" + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_nvidia_docker" # e.g. ray_docker # # Example of running a GPU head with CPU workers - # head_image: "rayproject/ray:latest-gpu" + # head_image: "rayproject/ray-ml:latest-gpu" - # worker_image: "rayproject/ray:latest" + # worker_image: "rayproject/ray-ml:latest" # If a node is idle for this many minutes, it will be removed. idle_timeout_minutes: 5 @@ -117,10 +118,10 @@ initialization_commands: done" # List of shell commands to run to set up nodes. -# NOTE: rayproject/ray:latest has ray latest bundled +# NOTE: rayproject/ray-ml:latest has ray latest bundled setup_commands: [] - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: diff --git a/python/ray/autoscaler/kubernetes/defaults.yaml b/python/ray/autoscaler/kubernetes/defaults.yaml index 8ba0b24fd..beadf1668 100644 --- a/python/ray/autoscaler/kubernetes/defaults.yaml +++ b/python/ray/autoscaler/kubernetes/defaults.yaml @@ -142,7 +142,7 @@ head_node: # - rsync (used for `ray rsync` commands and file mounts) # - screen (used for `ray attach`) # - kubectl (used by the autoscaler to manage worker pods) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] @@ -215,7 +215,7 @@ worker_nodes: # You are free (and encouraged) to use your own container image, # but it should have the following installed: # - rsync (used for `ray rsync` commands and file mounts) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] diff --git a/python/ray/autoscaler/kubernetes/example-full.yaml b/python/ray/autoscaler/kubernetes/example-full.yaml index 17f63f606..764048e1c 100644 --- a/python/ray/autoscaler/kubernetes/example-full.yaml +++ b/python/ray/autoscaler/kubernetes/example-full.yaml @@ -142,7 +142,7 @@ head_node: # - rsync (used for `ray rsync` commands and file mounts) # - screen (used for `ray attach`) # - kubectl (used by the autoscaler to manage worker pods) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] @@ -215,7 +215,7 @@ worker_nodes: # You are free (and encouraged) to use your own container image, # but it should have the following installed: # - rsync (used for `ray rsync` commands and file mounts) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] diff --git a/python/ray/autoscaler/kubernetes/example-ingress.yaml b/python/ray/autoscaler/kubernetes/example-ingress.yaml index 358dd09a3..47afaeff1 100644 --- a/python/ray/autoscaler/kubernetes/example-ingress.yaml +++ b/python/ray/autoscaler/kubernetes/example-ingress.yaml @@ -146,7 +146,7 @@ head_node: # - rsync (used for `ray rsync` commands and file mounts) # - screen (used for `ray attach`) # - kubectl (used by the autoscaler to manage worker pods) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] @@ -221,7 +221,7 @@ worker_nodes: # You are free (and encouraged) to use your own container image, # but it should have the following installed: # - rsync (used for `ray rsync` commands and file mounts) - image: rayproject/ray + image: rayproject/ray:nightly # Do not change this command - it keeps the pod alive until it is # explicitly killed. command: ["/bin/bash", "-c", "--"] diff --git a/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml b/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml new file mode 100644 index 000000000..9e92d5d4f --- /dev/null +++ b/python/ray/autoscaler/kubernetes/operator_configs/cluster_crd.yaml @@ -0,0 +1,4281 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rayclusters.cluster.ray.io +spec: + group: cluster.ray.io + scope: Namespaced + names: + plural: rayclusters + singular: raycluster + kind: RayCluster + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: Ray cluster configuration + type: object + required: + - spec + properties: + spec: + type: object + required: + - podTypes + - headPodType + - workerDefaultPodType + properties: + maxWorkers: + description: The maximum number of workers nodes to launch in addition to the + head node. + type: integer + minimum: 0 + upscalingSpeed: + description: The autoscaler will scale up the cluster faster with higher upscaling + speed. E.g., if the task requires adding more nodes then autoscaler will gradually + scale up the cluster in chunks of upscalingSpeed*currentlyRunningNodes. This + number should be > 0. + type: number + minimum: 0 + idleTimeoutMinutes: + description: If a node is idle for this many minutes, it will be removed. + type: integer + minimum: 0 + podTypes: + description: A list of Pod types on which to run Ray nodes, for multi-node-type autoscaling. + type: array + items: + type: object + required: + - name + - podConfig + properties: + name: + type: string + description: Name of the Pod type. + minWorkers: + type: integer + description: Minimum number of Ray workers of this Pod type. + maxWorkers: + type: integer + description: Maximum number of Ray workers of this Pod type. + rayResources: + type: object + description: User-specified custom resources for use by Ray. + # TODO (dmitri): Validate that values are numeric [patternProperties not supported by OpenAPI v3.0] + x-kubernetes-preserve-unknown-fields: true + setupCommands: + description: Commands to run before starting the Ray runtime. + type: array + items: + type: string + description: shell command + podConfig: + type: object + description: Pod configuration. + x-kubernetes-embedded-resource: true + properties: + spec: + description: PodSpec is a description of a pod. + properties: + activeDeadlineSeconds: + description: Optional duration in seconds the pod may be active + on the node relative to StartTime before the system will actively + try to mark it failed and kill associated containers. Value + must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a + no-op). A null preferred scheduling term matches + no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an + update), the system may or may not try to eventually + evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the + corresponding podAffinityTerm; the node(s) with the + highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the greatest + sum of weights, i.e. for each node that meets all + of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if the + node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: List of containers belonging to the pod. Containers + cannot currently be added or removed. There must be at least + one container in a Pod. Cannot be updated. + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or it's key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or it's key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images in + workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness probe failure, + preemption, resource contention, etc. The handler + is not called if the container crashes or exits. + The reason for termination is passed to the handler. + The Pod''s termination grace period countdown begins + before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but + is primarily informational. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" address + inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + default: TCP + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. + More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + type: object + stdin: + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. + This field is alpha in 1.14. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. + type: string + required: + - name + type: object + type: array + dnsConfig: + description: Specifies the DNS parameters of a pod. Parameters + specified here will be merged to the generated DNS configuration + based on DNSPolicy. + properties: + nameservers: + description: A list of DNS name server IP addresses. This + will be appended to the base nameservers generated from + DNSPolicy. Duplicated nameservers will be removed. + items: + type: string + type: array + options: + description: A list of DNS resolver options. This will be + merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options + given in Options will override those that appear in the + base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options + of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + searches: + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search paths + generated from DNSPolicy. Duplicated search paths will + be removed. + items: + type: string + type: array + type: object + dnsPolicy: + description: Set DNS policy for the pod. Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', + 'Default' or 'None'. DNS parameters given in DNSConfig will + be merged with the policy selected with DNSPolicy. To have + DNS options set along with hostNetwork, you have to specify + DNS policy explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' + type: boolean + hostAliases: + description: HostAliases is an optional list of hosts and IPs + that will be injected into the pod's hosts file if specified. + This is only valid for non-hostNetwork pods. + items: + description: HostAlias holds the mapping between IP and hostnames + that will be injected as an entry in the pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + hostIPC: + description: 'Use the host''s ipc namespace. Optional: Default + to false.' + type: boolean + hostNetwork: + description: Host networking requested for this pod. Use the + host's network namespace. If this option is set, the ports + that will be used must be specified. Default to false. + type: boolean + hostPID: + description: 'Use the host''s pid namespace. Optional: Default + to false.' + type: boolean + hostname: + description: Specifies the hostname of the Pod If not specified, + the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of references + to secrets in the same namespace to use for pulling any of + the images used by this PodSpec. If specified, these secrets + will be passed to individual puller implementations for them + to use. For example, in the case of docker, only DockerConfig + type secrets are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same + namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + initContainers: + description: 'List of initialization containers belonging to + the pod. Init containers are executed in order prior to containers + being started. If any init container fails, the pod is considered + to have failed and is handled according to its restartPolicy. + The name for an init container or normal container must be + unique among all containers. Init containers may not have + Lifecycle actions, Readiness probes, or Liveness probes. The + resourceRequirements of an init container are taken into account + during scheduling by finding the highest request/limit for + each resource type, and then using the max of of that value + or the sum of the normal containers. Limits are applied to + init containers in a similar fashion. Init containers cannot + currently be added or removed. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or it's key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or it's key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images in + workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness probe failure, + preemption, resource contention, etc. The handler + is not called if the container crashes or exits. + The reason for termination is passed to the handler. + The Pod''s termination grace period countdown begins + before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but + is primarily informational. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" address + inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + default: TCP + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to + perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness. Minimum value + is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. + More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + type: object + stdin: + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. + This field is alpha in 1.14. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. + type: string + required: + - name + type: object + type: array + nodeName: + description: NodeName is a request to schedule this pod onto + a specific node. If it is non-empty, the scheduler simply + schedules this pod onto that node, assuming that it fits resource + requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must be true + for the pod to fit on a node. Selector which must match a + node''s labels for the pod to be scheduled on that node. More + info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + priority: + description: The priority value. Various system components use + this field to find the priority of the pod. When Priority + Admission Controller is enabled, it prevents users from setting + this field. The admission controller populates this field + from PriorityClassName. The higher the value, the higher the + priority. + format: int32 + type: integer + priorityClassName: + description: If specified, indicates the pod's priority. "system-node-critical" + and "system-cluster-critical" are two special keywords which + indicate the highest priorities with the former being the + highest priority. Any other name must be defined by creating + a PriorityClass object with that name. If not specified, the + pod priority will be default or zero if there is no default. + type: string + readinessGates: + description: 'If specified, all readiness gates will be evaluated + for pod readiness. A pod is ready when all its containers + are ready AND all conditions specified in the readiness gates + have status equal to "True" More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' + items: + description: PodReadinessGate contains the reference to a + pod condition + properties: + conditionType: + description: ConditionType refers to a condition in the + pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + restartPolicy: + description: 'Restart policy for all containers within the pod. + One of Always, OnFailure, Never. Default to Always. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + type: string + runtimeClassName: + description: 'RuntimeClassName refers to a RuntimeClass object + in the node.k8s.io group, which should be used to run this + pod. If no RuntimeClass resource matches the named class, + the pod will not be run. If unset or empty, the "legacy" RuntimeClass + will be used, which is an implicit class with an empty definition + that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + This is an alpha feature and may change in the future.' + type: string + schedulerName: + description: If specified, the pod will be dispatched by specified + scheduler. If not specified, the pod will be dispatched by + default scheduler. + type: string + securityContext: + description: 'SecurityContext holds pod-level security attributes + and common container settings. Optional: Defaults to empty. See + type description for default values of each field.' + properties: + fsGroup: + description: "A special supplemental group that applies + to all containers in a pod. Some volume types allow the + Kubelet to change the ownership of that volume to be owned + by the pod: \n 1. The owning GID will be the FSGroup 2. + The setgid bit is set (new files created in the volume + will be owned by FSGroup) 3. The permission bits are OR'd + with rw-rw---- \n If unset, the Kubelet will not modify + the ownership and permissions of any volume." + format: int64 + type: integer + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a + non-root user. If true, the Kubelet will validate the + image at runtime to ensure that it does not run as UID + 0 (root) and fail to start the container if it does. If + unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence for + that container. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a + random SELinux context for each container. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + supplementalGroups: + description: A list of groups applied to the first process + run in each container, in addition to the container's + primary GID. If unspecified, no groups will be added + to any container. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used + for the pod. Pods with unsupported sysctls (by the container + runtime) might fail to launch. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + type: object + serviceAccount: + description: 'DeprecatedServiceAccount is a depreciated alias + for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' + type: string + serviceAccountName: + description: 'ServiceAccountName is the name of the ServiceAccount + to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + shareProcessNamespace: + description: 'Share a single process namespace between all of + the containers in a pod. When this is set containers will + be able to view and signal processes from other containers + in the same pod, and the first process in each container will + not be assigned PID 1. HostPID and ShareProcessNamespace cannot + both be set. Optional: Default to false. This field is beta-level + and may be disabled with the PodShareProcessNamespace feature.' + type: boolean + subdomain: + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have a domainname + at all. + type: string + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully. May be decreased in delete request. Value must + be non-negative integer. The value zero indicates delete immediately. + If this value is nil, the default grace period will be used + instead. The grace period is the duration in seconds after + the processes running in the pod are sent a termination signal + and the time when the processes are forcibly halted with a + kill signal. Set this value longer than the expected cleanup + time for your process. Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + volumes: + description: 'List of volumes that can be mounted by containers + belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the + ReadOnly property in VolumeMounts to "true". If + omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, + Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob + storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed + data disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the + host that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of + Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts. More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to + key ring for User, default is /etc/ceph/user.secret + More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to + the authentication secret for User, default is empty. + More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts. More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object + containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced ConfigMap will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If + a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked + optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or it's + keys must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + storage that is handled by an external CSI driver (Alpha + feature). + properties: + driver: + description: Driver is the name of the CSI driver + that handles this volume. Consult with your admin + for the correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the empty value + is passed to the associated CSI driver which will + determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all + secret references are passed. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for + the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about + the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of the + relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back + this directory. The default is "" which means to + use the node''s default medium. Must be an empty + string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum + of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + fc: + description: FC represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names + (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use + for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". The default filesystem depends + on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to + the secret object containing sensitive information + to pass to the plugin scripts. This may be empty + if no secret object is specified. If the secret + object contains more than one secret, all secrets + are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. + Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a + particular revision. DEPRECATED: GitRepo is deprecated. + To provision a container with a git repo, mount an EmptyDir + into an InitContainer that clones the repo using git, + then mount the EmptyDir into the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that + details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More + info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. + Defaults to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file + or directory on the host machine that is directly exposed + to the container. This is generally used for system + agents or other privileged things that are allowed to + see the host machine. Most containers will NOT need + this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'Path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to + "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://releases.k8s.io/HEAD/examples/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, + new iSCSI interface : + will be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal + is either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either + an IP or ip_addr:port if the port is other than + default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host + that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address + of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type + to mount Must be a filesystem type supported by + the host operating system. Ex. "ext4", "xfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by + default. Must be a value between 0 and 0777. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: information about the configMap + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed + keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or it's keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a + field of the pod: only annotations, + labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed + keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience + defaults to the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The + kubelet will start trying to rotate the + token if the token is older than 80 percent + of its time to live or if the token is + older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to + the mount point of the file to project + the token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the + host that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default + is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte + volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string + as host:port pair (multiple entries are separated + with commas) which acts as the central registry + for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned + Quobyte volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults + to serivceaccount user + type: string + volume: + description: Volume is a string that references an + already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for + RBDUser. Default is /etc/ceph/keyring. More info: + https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More + info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. + More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for + ScaleIO user and other sensitive information. If + this is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume + should be ThickProvisioned or ThinProvisioned. Default + is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with + the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created + in the ScaleIO system that is associated with this + volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced Secret will be + projected into the volume as a file whose name is + the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If + a key is specified which is not present in the Secret, + the volume setup will error unless it is marked + optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or it's keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use + for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name + of the StorageOS volume. Volume names are only + unique within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is + specified then the Pod's namespace will be used. This + allows the Kubernetes name scoping to be mirrored + within StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume + attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + + + headPodType: + description: Specifies the head node type. + type: string + workerDefaultPodType: + description: Specifies the default worker node type. + type: string + headStartRayCommands: + description: Commands to start Ray on the head node. + type: array + items: + type: string + description: shell command + workerStartRayCommands: + description: Commands to start Ray on worker nodes. + type: array + items: + type: string + description: shell command diff --git a/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml b/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml new file mode 100644 index 000000000..bb4a71fcc --- /dev/null +++ b/python/ray/autoscaler/kubernetes/operator_configs/example_cluster.yaml @@ -0,0 +1,128 @@ +apiVersion: cluster.ray.io/v1 +kind: RayCluster +metadata: + name: example-cluster +spec: + # The maximum number of workers nodes to launch in addition to the head node. + maxWorkers: 3 + # The autoscaler will scale up the cluster faster with higher upscaling speed. + # E.g., if the task requires adding more nodes then autoscaler will gradually + # scale up the cluster in chunks of upscaling_speed*currently_running_nodes. + # This number should be > 0. + upscalingSpeed: 1.0 + # If a node is idle for this many minutes, it will be removed. + idleTimeoutMinutes: 5 + # Specify the pod type for the ray head node (as configured below). + headPodType: head-node + # Specify the default pod type for ray the worker nodes (as configured below). + workerDefaultPodType: worker-nodes + # Specify the allowed pod types for this ray cluster and the resources they provide. + podTypes: + - name: head-node + podConfig: + apiVersion: v1 + kind: Pod + metadata: + # Automatically generates a name for the pod with this prefix. + generateName: example-cluster-ray-head- + spec: + restartPolicy: Never + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-node + imagePullPolicy: Always + image: rayproject/ray:nightly + # Do not change this command - it keeps the pod alive until it is + # explicitly killed. + command: ["/bin/bash", "-c", "--"] + args: ['trap : TERM INT; sleep infinity & wait;'] + ports: + - containerPort: 6379 # Redis port. + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 1000m + memory: 512Mi + limits: + # The maximum memory that this pod is allowed to use. The + # limit will be detected by ray and split to use 10% for + # redis, 30% for the shared memory object store, and the + # rest for application memory. If this limit is not set and + # the object store size is not set manually, ray will + # allocate a very large object store in each pod that may + # cause problems for other pods. + memory: 512Mi + - name: worker-nodes + # Minimum number of Ray workers of this Pod type. + minWorkers: 2 + # Maximum number of Ray workers of this Pod type. Takes precedence over minWorkers. + maxWorkers: 3 + # User-specified custom resources for use by Ray + rayResources: {"Custom1": 1, "is_spot": 1} + # Optional commands to run before starting the Ray runtime. + setupCommands: + - pip install numpy # Example + podConfig: + apiVersion: v1 + kind: Pod + metadata: + # Automatically generates a name for the pod with this prefix. + generateName: example-cluster-ray-worker- + spec: + restartPolicy: Never + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-node + imagePullPolicy: Always + image: rayproject/ray:nightly + command: ["/bin/bash", "-c", "--"] + args: ["trap : TERM INT; sleep infinity & wait;"] + ports: + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 1000m + memory: 512Mi + limits: + # The maximum memory that this pod is allowed to use. The + # limit will be detected by ray and split to use 10% for + # redis, 30% for the shared memory object store, and the + # rest for application memory. If this limit is not set and + # the object store size is not set manually, ray will + # allocate a very large object store in each pod that may + # cause problems for other pods. + memory: 512Mi + # Commands to start Ray on the head node. You don't need to change this. + # Note dashboard-host is set to 0.0.0.0 so that Kubernetes can port forward. + headStartRayCommands: + - ray stop + - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --dashboard-host 0.0.0.0 + # Commands to start Ray on worker nodes. You don't need to change this. + workerStartRayCommands: + - ray stop + - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml b/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml new file mode 100644 index 000000000..e5e4ecf31 --- /dev/null +++ b/python/ray/autoscaler/kubernetes/operator_configs/example_cluster2.yaml @@ -0,0 +1,128 @@ +apiVersion: cluster.ray.io/v1 +kind: RayCluster +metadata: + name: example-cluster2 +spec: + # The maximum number of workers nodes to launch in addition to the head node. + maxWorkers: 3 + # The autoscaler will scale up the cluster faster with higher upscaling speed. + # E.g., if the task requires adding more nodes then autoscaler will gradually + # scale up the cluster in chunks of upscaling_speed*currently_running_nodes. + # This number should be > 0. + upscalingSpeed: 1.0 + # If a node is idle for this many minutes, it will be removed. + idleTimeoutMinutes: 5 + # Specify the pod type for the ray head node (as configured below). + headPodType: head-node + # Specify the default pod type for ray the worker nodes (as configured below). + workerDefaultPodType: worker-nodes + # Specify the allowed pod types for this ray cluster and the resources they provide. + podTypes: + - name: head-node + podConfig: + apiVersion: v1 + kind: Pod + metadata: + # Automatically generates a name for the pod with this prefix. + generateName: example-cluster2-ray-head- + spec: + restartPolicy: Never + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-node + imagePullPolicy: Always + image: rayproject/ray:nightly + # Do not change this command - it keeps the pod alive until it is + # explicitly killed. + command: ["/bin/bash", "-c", "--"] + args: ['trap : TERM INT; sleep infinity & wait;'] + ports: + - containerPort: 6379 # Redis port. + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 1000m + memory: 512Mi + limits: + # The maximum memory that this pod is allowed to use. The + # limit will be detected by ray and split to use 10% for + # redis, 30% for the shared memory object store, and the + # rest for application memory. If this limit is not set and + # the object store size is not set manually, ray will + # allocate a very large object store in each pod that may + # cause problems for other pods. + memory: 512Mi + - name: worker-nodes + # Minimum number of Ray workers of this Pod type. + minWorkers: 1 + # Maximum number of Ray workers of this Pod type. Takes precedence over minWorkers. + maxWorkers: 3 + # User-specified custom resources for use by Ray + rayResources: {"Custom1": 1, "is_spot": 1} + # Optional commands to run before starting the Ray runtime. + setupCommands: + - pip install numpy # Example + podConfig: + apiVersion: v1 + kind: Pod + metadata: + # Automatically generates a name for the pod with this prefix. + generateName: example-cluster2-ray-worker- + spec: + restartPolicy: Never + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: ray-node + imagePullPolicy: Always + image: rayproject/ray:nightly + command: ["/bin/bash", "-c", "--"] + args: ["trap : TERM INT; sleep infinity & wait;"] + ports: + - containerPort: 12345 # Ray internal communication. + - containerPort: 12346 # Ray internal communication. + # This volume allocates shared memory for Ray to use for its plasma + # object store. If you do not provide this, Ray will fall back to + # /tmp which cause slowdowns if is not a shared memory volume. + volumeMounts: + - mountPath: /dev/shm + name: dshm + resources: + requests: + cpu: 1000m + memory: 512Mi + limits: + # The maximum memory that this pod is allowed to use. The + # limit will be detected by ray and split to use 10% for + # redis, 30% for the shared memory object store, and the + # rest for application memory. If this limit is not set and + # the object store size is not set manually, ray will + # allocate a very large object store in each pod that may + # cause problems for other pods. + memory: 512Mi + # Commands to start Ray on the head node. You don't need to change this. + # Note dashboard-host is set to 0.0.0.0 so that Kubernetes can port forward. + headStartRayCommands: + - ray stop + - ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --dashboard-host 0.0.0.0 + # Commands to start Ray on worker nodes. You don't need to change this. + workerStartRayCommands: + - ray stop + - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml b/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml similarity index 78% rename from python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml rename to python/ray/autoscaler/kubernetes/operator_configs/operator.yaml index 5cd239fac..956f27ff7 100644 --- a/python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml +++ b/python/ray/autoscaler/kubernetes/operator_configs/operator.yaml @@ -9,8 +9,8 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ray-operator-role rules: -- apiGroups: ["", "rbac.authorization.k8s.io"] - resources: ["configmaps", "pods", "pods/exec", "services", "serviceaccounts", "roles", "rolebindings"] +- apiGroups: ["", "cluster.ray.io"] + resources: ["rayclusters", "pods", "pods/exec"] verbs: ["get", "watch", "list", "create", "delete", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -35,8 +35,7 @@ spec: - name: ray imagePullPolicy: Always image: rayproject/ray:nightly - command: ["/bin/bash", "-c", "--"] - args: ["ray-operator; trap : TERM INT; sleep infinity & wait;"] + command: ["ray-operator"] env: - name: RAY_OPERATOR_POD_NAMESPACE valueFrom: diff --git a/python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml b/python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml deleted file mode 100644 index d42e5749d..000000000 --- a/python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml +++ /dev/null @@ -1,260 +0,0 @@ -# An unique identifier for the head node and workers of this cluster. -cluster_name: default - -# The autoscaler will scale up the cluster to this target fraction of resource -# usage. For example, if a cluster of 10 nodes is 100% busy and -# target_utilization is 0.8, it would resize the cluster to 13. This fraction -# can be decreased to increase the aggressiveness of upscaling. -# This value must be less than 1.0 for scaling to happen. -target_utilization_fraction: 0.8 - -# If a node is idle for this many minutes, it will be removed. -idle_timeout_minutes: 5 - -# Kubernetes resources that need to be configured for the autoscaler to be -# able to manage the Ray cluster. If any of the provided resources don't -# exist, the autoscaler will attempt to create them. If this fails, you may -# not have the required permissions and will have to request them to be -# created by your cluster administrator. -provider: - type: kubernetes - - # Exposing external IP addresses for ray pods isn't currently supported. - use_internal_ips: true - - # Namespace to use for all resources created. - namespace: ray - - services: - # Service that maps to the head node of the Ray cluster. - - apiVersion: v1 - kind: Service - metadata: - # NOTE: If you're running multiple Ray clusters with services - # on one Kubernetes cluster, they must have unique service - # names. - name: ray-head - spec: - # This selector must match the head node pod's selector below. - selector: - component: ray-head - ports: - - protocol: TCP - port: 8000 - targetPort: 8000 - - # Service that maps to the worker nodes of the Ray cluster. - - apiVersion: v1 - kind: Service - metadata: - # NOTE: If you're running multiple Ray clusters with services - # on one Kubernetes cluster, they must have unique service - # names. - name: ray-workers - spec: - # This selector must match the worker node pods' selector below. - selector: - component: ray-worker - ports: - - protocol: TCP - port: 8000 - targetPort: 8000 - -# Kubernetes pod config for the head node pod. -available_node_types: - head_node: - resources: {} - node_config: - apiVersion: v1 - kind: Pod - metadata: - # Automatically generates a name for the pod with this prefix. - generateName: ray-head- - - # Must match the head node service selector above if a head node - # service is required. - labels: - component: ray-head - spec: - # Restarting the head node automatically is not currently supported. - # If the head node goes down, `ray up` must be run again. - restartPolicy: Never - - # This volume allocates shared memory for Ray to use for its plasma - # object store. If you do not provide this, Ray will fall back to - # /tmp which cause slowdowns if is not a shared memory volume. - volumes: - - name: dshm - emptyDir: - medium: Memory - - containers: - - name: ray-node - imagePullPolicy: Always - # You are free (and encouraged) to use your own container image, - # but it should have the following installed: - # - rsync (used for `ray rsync` commands and file mounts) - # - screen (used for `ray attach`) - # - kubectl (used by the autoscaler to manage worker pods) - image: rayproject/ray:nightly - # Do not change this command - it keeps the pod alive until it is - # explicitly killed. - command: ["/bin/bash", "-c", "--"] - args: ["trap : TERM INT; sleep infinity & wait;"] - ports: - - containerPort: 6379 # Redis port. - - containerPort: 6380 # Redis port. - - containerPort: 6381 # Redis port. - - containerPort: 12345 # Ray internal communication. - - containerPort: 12346 # Ray internal communication. - - # This volume allocates shared memory for Ray to use for its plasma - # object store. If you do not provide this, Ray will fall back to - # /tmp which cause slowdowns if is not a shared memory volume. - volumeMounts: - - mountPath: /dev/shm - name: dshm - resources: - requests: - cpu: 1000m - memory: 512Mi - limits: - # The maximum memory that this pod is allowed to use. The - # limit will be detected by ray and split to use 10% for - # redis, 30% for the shared memory object store, and the - # rest for application memory. If this limit is not set and - # the object store size is not set manually, ray will - # allocate a very large object store in each pod that may - # cause problems for other pods. - memory: 2Gi - env: - # This is used in the head_start_ray_commands below so that - # Ray can spawn the correct number of processes. Omitting this - # may lead to degraded performance. - - name: MY_CPU_REQUEST - valueFrom: - resourceFieldRef: - resource: requests.cpu - - worker_nodes: - resources: {} - min_workers: 1 - max_workers: 2 - node_config: - apiVersion: v1 - kind: Pod - metadata: - # Automatically generates a name for the pod with this prefix. - generateName: ray-worker- - - # Must match the worker node service selector above if a worker node - # service is required. - labels: - component: ray-worker - spec: - serviceAccountName: default - - # Worker nodes will be managed automatically by the head node, so - # do not change the restart policy. - restartPolicy: Never - - # This volume allocates shared memory for Ray to use for its plasma - # object store. If you do not provide this, Ray will fall back to - # /tmp which cause slowdowns if is not a shared memory volume. - volumes: - - name: dshm - emptyDir: - medium: Memory - - containers: - - name: ray-node - imagePullPolicy: Always - # You are free (and encouraged) to use your own container image, - # but it should have the following installed: - # - rsync (used for `ray rsync` commands and file mounts) - image: rayproject/ray:nightly - # Do not change this command - it keeps the pod alive until it is - # explicitly killed. - command: ["/bin/bash", "-c", "--"] - args: ["trap : TERM INT; sleep infinity & wait;"] - ports: - - containerPort: 12345 # Ray internal communication. - - containerPort: 12346 # Ray internal communication. - - # This volume allocates shared memory for Ray to use for its plasma - # object store. If you do not provide this, Ray will fall back to - # /tmp which cause slowdowns if is not a shared memory volume. - volumeMounts: - - mountPath: /dev/shm - name: dshm - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - # This memory limit will be detected by ray and split into - # 30% for plasma, and 70% for workers. - memory: 2Gi - env: - # This is used in the head_start_ray_commands below so that - # Ray can spawn the correct number of processes. Omitting this - # may lead to degraded performance. - - name: MY_CPU_REQUEST - valueFrom: - resourceFieldRef: - resource: requests.cpu - -head_node_type: - head_node - -worker_default_node_type: - worker_nodes -# Files or directories to copy to the head and worker nodes. The format is a -# dictionary from REMOTE_PATH: LOCAL_PATH, e.g. -file_mounts: { -} - -# Files or directories to copy from the head node to the worker nodes. The format is a -# list of paths. The same path on the head node will be copied to the worker node. -# This behavior is a subset of the file_mounts behavior. In the vast majority of cases -# you should just use file_mounts. Only use this if you know what you're doing! -cluster_synced_files: [] - -# Whether changes to directories in file_mounts or cluster_synced_files in the head node -# should sync to the worker node continuously -file_mounts_sync_continuously: False - -# Patterns for files to exclude when running rsync up or rsync down. -# This is not supported on kubernetes. -rsync_exclude: [] - -# Pattern files to use for filtering out files when running rsync up or rsync down. The file is searched for -# in the source directory and recursively through all subdirectories. For example, if .gitignore is provided -# as a value, the behavior will match git's behavior for finding and using .gitignore files. -# This is not supported on kubernetes. -rsync_filter: [] - -# List of commands that will be run before `setup_commands`. If docker is -# enabled, these commands will run outside the container and before docker -# is setup. -initialization_commands: [] - -# List of shell commands to run to set up nodes. -setup_commands: [] - -# Custom commands that will be run on the head node after common setup. -head_setup_commands: [] - -# Custom commands that will be run on worker nodes after common setup. -worker_setup_commands: [] - -# Command to start ray on the head node. You don't need to change this. -# Note webui-host is set to 0.0.0.0 so that kubernetes can port forward. -head_start_ray_commands: - - ray stop - - ulimit -n 65536; ray start --head --num-cpus=$MY_CPU_REQUEST --object-manager-port=8076 --dashboard-host 0.0.0.0 - -# Command to start ray on worker nodes. You don't need to change this. -worker_start_ray_commands: - - ray stop - - ulimit -n 65536; ray start --num-cpus=$MY_CPU_REQUEST --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/python/ray/autoscaler/local/example-full.yaml b/python/ray/autoscaler/local/example-full.yaml index ff0d651d0..4d93191fd 100644 --- a/python/ray/autoscaler/local/example-full.yaml +++ b/python/ray/autoscaler/local/example-full.yaml @@ -25,7 +25,8 @@ idle_timeout_minutes: 5 # and opens all the necessary ports to support the Ray cluster. # Empty string means disabled. Assumes Docker is installed. docker: - image: "rayproject/ray:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + image: "rayproject/ray-ml:latest-gpu" # You can change this to latest-cpu if you don't need GPU support and want a faster startup + # image: rayproject/ray:latest-gpu # use this one if you don't need ML dependencies, it's faster to pull container_name: "ray_container" # If true, pulls latest version of image. Otherwise, `docker run` will only pull the image # if no cached version is present. @@ -93,7 +94,7 @@ setup_commands: [] # has your Ray repo pre-cloned. Then, you can replace the pip installs # below with a git checkout (and possibly a recompile). # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest) - # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Custom commands that will be run on the head node after common setup. head_setup_commands: [] diff --git a/python/ray/autoscaler/ray-schema.json b/python/ray/autoscaler/ray-schema.json index 0f6c3bd52..41a4a0708 100644 --- a/python/ray/autoscaler/ray-schema.json +++ b/python/ray/autoscaler/ray-schema.json @@ -20,7 +20,7 @@ "additionalProperties": false, "properties": { "cluster_name": { - "description": "An unique identifier for the head node and workers of this cluster.", + "description": "A unique identifier for the head node and workers of this cluster.", "type": "string" }, "min_workers": { diff --git a/python/ray/exceptions.py b/python/ray/exceptions.py index 0456a3a6d..b5a0b477c 100644 --- a/python/ray/exceptions.py +++ b/python/ray/exceptions.py @@ -3,9 +3,8 @@ from traceback import format_exception import colorama -import ray import ray.cloudpickle as pickle -from ray.core.generated.common_pb2 import RayException, Language +from ray.core.generated.common_pb2 import RayException, Language, PYTHON import setproctitle @@ -17,7 +16,7 @@ class RayError(Exception): exc_info = (type(self), self, self.__traceback__) formatted_exception_string = "\n".join(format_exception(*exc_info)) return RayException( - language=ray.Language.PYTHON.value(), + language=PYTHON, serialized_exception=pickle.dumps(self), formatted_exception_string=formatted_exception_string ).SerializeToString() @@ -26,7 +25,7 @@ class RayError(Exception): def from_bytes(b): ray_exception = RayException() ray_exception.ParseFromString(b) - if ray_exception.language == ray.Language.PYTHON.value(): + if ray_exception.language == PYTHON: return pickle.loads(ray_exception.serialized_exception) else: return CrossLanguageError(ray_exception) @@ -81,6 +80,7 @@ class RayTaskError(RayError): pid=None, ip=None): """Initialize a RayTaskError.""" + import ray if proctitle: self.proctitle = proctitle else: diff --git a/python/ray/experimental/client/__init__.py b/python/ray/experimental/client/__init__.py index 36d19ba56..a6bba39ed 100644 --- a/python/ray/experimental/client/__init__.py +++ b/python/ray/experimental/client/__init__.py @@ -7,34 +7,88 @@ import logging logger = logging.getLogger(__name__) -# _client_api has to be external to the API stub, below. -# Otherwise, ray.remote() that contains ray.remote() -# contains a reference to the RayAPIStub, therefore a -# reference to the _client_api, and then tries to pickle -# the thing. +# About these global variables: Ray 1.0 uses exported module functions to +# provide its API, and we need to match that. However, we want different +# behaviors depending on where, exactly, in the client stack this is running. +# +# The reason for these differences depends on what's being pickled and passed +# to functions, or functions inside functions. So there are three cases to care +# about +# +# (Python Client)-->(Python ClientServer)-->(Internal Raylet Process) +# +# * _client_api should be set if we're inside the client +# * _server_api should be set if we're inside the clientserver +# * Both will be set if we're running both (as in a test) +# * Neither should be set if we're inside the raylet (but we still need to shim +# from the client API surface to the Ray API) +# +# The job of RayAPIStub (below) delegates to the appropriate one of these +# depending on what's set or not. Then, all users importing the ray object +# from this package get the stub which routes them to the appropriate APIImpl. _client_api: Optional[APIImpl] = None +_server_api: Optional[APIImpl] = None + +# The reason for _is_server is a hack around the above comment while running +# tests. If we have both a client and a server trying to control these static +# variables then we need a way to decide which to use. In this case, both +# _client_api and _server_api are set. +# This boolean flips between the two +_is_server: bool = False @contextmanager def stash_api_for_tests(in_test: bool): - api = None + global _is_server + is_server = _is_server if in_test: - api = stash_api() - yield api + _is_server = True + yield _server_api if in_test: - restore_api(api) + _is_server = is_server -def stash_api() -> Optional[APIImpl]: +def _set_client_api(val: Optional[APIImpl]): global _client_api - a = _client_api + global _is_server + if _client_api is not None: + raise Exception("Trying to set more than one client API") + _client_api = val + _is_server = False + + +def _set_server_api(val: Optional[APIImpl]): + global _server_api + global _is_server + if _server_api is not None: + raise Exception("Trying to set more than one server API") + _server_api = val + _is_server = True + + +def reset_api(): + global _client_api + global _server_api + global _is_server _client_api = None - return a + _server_api = None + _is_server = False -def restore_api(api: Optional[APIImpl]): +def _get_client_api() -> APIImpl: global _client_api - _client_api = api + global _server_api + global _is_server + api = None + if _is_server: + api = _server_api + else: + api = _client_api + if api is None: + # We're inside a raylet worker + from ray.experimental.client.server.core_ray_api import CoreRayAPI + return CoreRayAPI() + return api class RayAPIStub: @@ -43,11 +97,10 @@ class RayAPIStub: secure: bool = False, metadata: List[Tuple[str, str]] = None, stub=None): - global _client_api from ray.experimental.client.worker import Worker _client_worker = Worker( conn_str, secure=secure, metadata=metadata, stub=stub) - _client_api = ClientAPI(_client_worker) + _set_client_api(ClientAPI(_client_worker)) def disconnect(self): global _client_api @@ -56,15 +109,9 @@ class RayAPIStub: _client_api = None def __getattr__(self, key: str): - global _client_api - self.__check_client_api() - return getattr(_client_api, key) - - def __check_client_api(self): - global _client_api - if _client_api is None: - from ray.experimental.client.server.core_ray_api import CoreRayAPI - _client_api = CoreRayAPI() + global _get_client_api + api = _get_client_api() + return getattr(api, key) ray = RayAPIStub() diff --git a/python/ray/experimental/client/api.py b/python/ray/experimental/client/api.py index 17d0d6a97..304cc4467 100644 --- a/python/ray/experimental/client/api.py +++ b/python/ray/experimental/client/api.py @@ -11,40 +11,145 @@ from abc import ABC from abc import abstractmethod +from typing import TYPE_CHECKING, Any, Union, Optional +import ray.core.generated.ray_client_pb2 as ray_client_pb2 +if TYPE_CHECKING: + from ray.experimental.client.common import ClientActorHandle + from ray.experimental.client.common import ClientStub + from ray.experimental.client.common import ClientObjectRef + from ray._raylet import ObjectRef + + # Use the imports for type checking. This is a python 3.6 limitation. + # See https://www.python.org/dev/peps/pep-0563/ + PutType = Union[ClientObjectRef, ObjectRef] class APIImpl(ABC): + """ + APIImpl is the interface to implement for whichever version of the core + Ray API that needs abstracting when run in client mode. + """ + @abstractmethod - def get(self, *args, **kwargs): + def get(self, vals, *, timeout: Optional[float] = None) -> Any: + """ + get is the hook stub passed on to replace `ray.get` + + Args: + vals: [Client]ObjectRef or list of these refs to retrieve. + timeout: Optional timeout in milliseconds + """ pass @abstractmethod - def put(self, *args, **kwargs): + def put(self, vals: Any, *args, + **kwargs) -> Union["ClientObjectRef", "ObjectRef"]: + """ + put is the hook stub passed on to replace `ray.put` + + Args: + vals: The value or list of values to `put`. + args: opaque arguments + kwargs: opaque keyword arguments + """ pass @abstractmethod def wait(self, *args, **kwargs): + """ + wait is the hook stub passed on to replace `ray.wait` + + Args: + args: opaque arguments + kwargs: opaque keyword arguments + """ pass @abstractmethod def remote(self, *args, **kwargs): + """ + remote is the hook stub passed on to replace `ray.remote`. + + This sets up remote functions or actors, as the decorator, + but does not execute them. + + Args: + args: opaque arguments + kwargs: opaque keyword arguments + """ pass @abstractmethod - def call_remote(self, f, kind, *args, **kwargs): + def call_remote(self, instance: "ClientStub", *args, **kwargs): + """ + call_remote is called by stub objects to execute them remotely. + + This is used by stub objects in situations where they're called + with .remote, eg, `f.remote()` or `actor_cls.remote()`. + This allows the client stub objects to delegate execution to be + implemented in the most effective way whether it's in the client, + clientserver, or raylet worker. + + Args: + instance: The Client-side stub reference to a remote object + args: opaque arguments + kwargs: opaque keyword arguments + """ pass @abstractmethod - def close(self, *args, **kwargs): + def close(self) -> None: + """ + close cleans up an API connection by closing any channels or + shutting down any servers gracefully. + """ + pass + + @abstractmethod + def kill(self, actor, *, no_restart=True): + """ + kill forcibly stops an actor running in the cluster + + Args: + no_restart: Whether this actor should be restarted if it's a + restartable actor. + """ + pass + + @abstractmethod + def cancel(self, obj, *, force=False, recursive=True): + """ + Cancels a task on the cluster. + + If the specified task is pending execution, it will not be executed. If + the task is currently executing, the behavior depends on the ``force`` + flag, as per `ray.cancel()` + + Only non-actor tasks can be canceled. Canceled tasks will not be + retried (max_retries will not be respected). + + Args: + object_ref (ObjectRef): ObjectRef returned by the task + that should be canceled. + force (boolean): Whether to force-kill a running task by killing + the worker that is running the task. + recursive (boolean): Whether to try to cancel tasks submitted by + the task specified. + """ pass class ClientAPI(APIImpl): + """ + The Client-side methods corresponding to the ray API. Delegates + to the Client Worker that contains the connection to the ClientServer. + """ + def __init__(self, worker): self.worker = worker - def get(self, *args, **kwargs): - return self.worker.get(*args, **kwargs) + def get(self, vals, *, timeout=None): + return self.worker.get(vals, timeout=timeout) def put(self, *args, **kwargs): return self.worker.put(*args, **kwargs) @@ -55,12 +160,65 @@ class ClientAPI(APIImpl): def remote(self, *args, **kwargs): return self.worker.remote(*args, **kwargs) - def call_remote(self, f, kind, *args, **kwargs): - return self.worker.call_remote(f, kind, *args, **kwargs) + def call_remote(self, instance: "ClientStub", *args, **kwargs): + return self.worker.call_remote(instance, *args, **kwargs) - def close(self, *args, **kwargs): + def close(self) -> None: return self.worker.close() + def kill(self, actor: "ClientActorHandle", *, no_restart=True): + return self.worker.terminate_actor(actor, no_restart) + + def cancel(self, obj: "ClientObjectRef", *, force=False, recursive=True): + return self.worker.terminate_task(obj, force, recursive) + + # Various metadata methods for the client that are defined in the protocol. + def is_initialized(self) -> bool: + """ True if our client is connected, and if the server is initialized. + + Returns: + A boolean determining if the client is connected and + server initialized. + """ + return self.worker.is_initialized() + + def nodes(self): + """Get a list of the nodes in the cluster (for debugging only). + + Returns: + Information about the Ray clients in the cluster. + """ + return self.worker.get_cluster_info( + ray_client_pb2.ClusterInfoType.NODES) + + def cluster_resources(self): + """Get the current total cluster resources. + + Note that this information can grow stale as nodes are added to or + removed from the cluster. + + Returns: + A dictionary mapping resource name to the total quantity of that + resource in the cluster. + """ + return self.worker.get_cluster_info( + ray_client_pb2.ClusterInfoType.CLUSTER_RESOURCES) + + def available_resources(self): + """Get the current available cluster resources. + + This is different from `cluster_resources` in that this will return + idle (available) resources rather than total resources. + + Note that this information can grow stale as tasks start and finish. + + Returns: + A dictionary mapping resource name to the total quantity of that + resource in the cluster. + """ + return self.worker.get_cluster_info( + ray_client_pb2.ClusterInfoType.AVAILABLE_RESOURCES) + def __getattr__(self, key: str): if not key.startswith("_"): raise NotImplementedError( diff --git a/python/ray/experimental/client/common.py b/python/ray/experimental/client/common.py index d2ec7e041..24b012790 100644 --- a/python/ray/experimental/client/common.py +++ b/python/ray/experimental/client/common.py @@ -1,12 +1,16 @@ import ray.core.generated.ray_client_pb2 as ray_client_pb2 from ray.experimental.client import ray from typing import Any +from typing import Dict from ray import cloudpickle +import base64 + class ClientBaseRef: - def __init__(self, id): + def __init__(self, id, handle=None): self.id = id + self.handle = handle def __repr__(self): return "%s(%s)" % ( @@ -17,83 +21,243 @@ class ClientBaseRef: def __eq__(self, other): return self.id == other.id + def binary(self): + return self.id + + @classmethod + def from_remote_ref(cls, ref: ray_client_pb2.RemoteRef): + return cls(id=ref.id, handle=ref.handle) + class ClientObjectRef(ClientBaseRef): - pass + def _unpack_ref(self): + return cloudpickle.loads(self.handle) class ClientActorRef(ClientBaseRef): pass -class ClientRemoteFunc: +class ClientStub: + pass + + +class ClientRemoteFunc(ClientStub): + """ + A stub created on the Ray Client to represent a remote + function that can be exectued on the cluster. + + This class is allowed to be passed around between remote functions. + + Args: + _func: The actual function to execute remotely + _name: The original name of the function + _ref: The ClientObjectRef of the pickled code of the function, _func + _raylet_remote: The Raylet-side ray.remote_function.RemoteFunction + for this object + """ + def __init__(self, f): self._func = f self._name = f.__name__ self.id = None - self._raylet_remote_func = None + + # self._ref can be lazily instantiated. Rather than eagerly creating + # function data objects in the server we can put them just before we + # execute the function, especially in cases where many @ray.remote + # functions exist in a library and only a handful are ever executed by + # a user of the library. + # + # TODO(barakmich): This ref might actually be better as a serialized + # ObjectRef. This requires being able to serialize the ref without + # pinning it (as the lifetime of the ref is tied with the server, not + # the client) + self._ref = None + self._raylet_remote = None def __call__(self, *args, **kwargs): raise TypeError(f"Remote function cannot be called directly. " "Use {self._name}.remote method instead") def remote(self, *args, **kwargs): - return ray.call_remote(self, ray_client_pb2.ClientTask.FUNCTION, *args, - **kwargs) + return ray.call_remote(self, *args, **kwargs) + + def _get_ray_remote_impl(self): + if self._raylet_remote is None: + self._raylet_remote = ray.remote(self._func) + return self._raylet_remote def __repr__(self): - return "ClientRemoteFunc(%s, %s)" % (self._name, self.id) + return "ClientRemoteFunc(%s, %s)" % (self._name, self._ref) + + def _prepare_client_task(self) -> ray_client_pb2.ClientTask: + if self._ref is None: + self._ref = ray.put(self._func) + task = ray_client_pb2.ClientTask() + task.type = ray_client_pb2.ClientTask.FUNCTION + task.name = self._name + task.payload_id = self._ref.handle + return task -class ClientActorClass: +class ClientActorClass(ClientStub): + """ A stub created on the Ray Client to represent an actor class. + + It is wrapped by ray.remote and can be executed on the cluster. + + Args: + actor_cls: The actual class to execute remotely + _name: The original name of the class + _ref: The ClientObjectRef of the pickled `actor_cls` + _raylet_remote: The Raylet-side ray.ActorClass for this object + """ + def __init__(self, actor_cls): self.actor_cls = actor_cls self._name = actor_cls.__name__ + self._ref = None + self._raylet_remote = None def __call__(self, *args, **kwargs): raise TypeError(f"Remote actor cannot be instantiated directly. " "Use {self._name}.remote() instead") + def __getstate__(self) -> Dict: + state = { + "actor_cls": self.actor_cls, + "_name": self._name, + "_ref": self._ref, + } + return state + + def __setstate__(self, state: Dict) -> None: + self.actor_cls = state["actor_cls"] + self._name = state["_name"] + self._ref = state["_ref"] + def remote(self, *args, **kwargs): # Actually instantiate the actor - ref = ray.call_remote(self, ray_client_pb2.ClientTask.ACTOR, *args, - **kwargs) - return ClientActorHandle(ref, self) + ref = ray.call_remote(self, *args, **kwargs) + return ClientActorHandle(ClientActorRef(ref.id, ref.handle), self) def __repr__(self): - return "ClientRemoteActor(%s, %s)" % (self._name, self.id) + return "ClientRemoteActor(%s, %s)" % (self._name, self._ref) def __getattr__(self, key): + if key not in self.__dict__: + raise AttributeError("Not a class attribute") raise NotImplementedError("static methods") + def _prepare_client_task(self) -> ray_client_pb2.ClientTask: + if self._ref is None: + self._ref = ray.put(self.actor_cls) + task = ray_client_pb2.ClientTask() + task.type = ray_client_pb2.ClientTask.ACTOR + task.name = self._name + task.payload_id = self._ref.handle + return task -class ClientActorHandle: - def __init__(self, actor_id: ClientActorRef, + +class ClientActorHandle(ClientStub): + """Client-side stub for instantiated actor. + + A stub created on the Ray Client to represent a remote actor that + has been started on the cluster. This class is allowed to be passed + around between remote functions. + + Args: + actor_ref: A reference to the running actor given to the client. This + is a serialized version of the actual handle as an opaque token. + actor_class: A reference to the ClientActorClass that this actor was + instantiated from. + _real_actor_handle: Cached copy of the Raylet-side + ray.actor.ActorHandle contained in the actor_id ref. + """ + + def __init__(self, actor_ref: ClientActorRef, actor_class: ClientActorClass): - self.actor_id = actor_id + self.actor_ref = actor_ref self.actor_class = actor_class + self._real_actor_handle = None + + def _get_ray_remote_impl(self): + if self._real_actor_handle is None: + self._real_actor_handle = cloudpickle.loads(self.actor_ref.handle) + return self._real_actor_handle + + def __getstate__(self) -> Dict: + state = { + "actor_ref": self.actor_ref, + "actor_class": self.actor_class, + "_real_actor_handle": self._real_actor_handle, + } + return state + + def __setstate__(self, state: Dict) -> None: + self.actor_ref = state["actor_ref"] + self.actor_class = state["actor_class"] + self._real_actor_handle = state["_real_actor_handle"] + + @property + def _actor_id(self): + return self.actor_ref.id def __getattr__(self, key): return ClientRemoteMethod(self, key) + def __repr__(self): + return "ClientActorHandle(%s)" % (self.actor_ref.id.hex()) + + +class ClientRemoteMethod(ClientStub): + """A stub for a method on a remote actor. + + Can be annotated with exection options. + + Args: + actor_handle: A reference to the ClientActorHandle that generated + this method and will have this method called upon it. + method_name: The name of this method + """ -class ClientRemoteMethod: def __init__(self, actor_handle: ClientActorHandle, method_name: str): self.actor_handle = actor_handle self.method_name = method_name - self._name = "%s.%s" % (self.actor_handle.actor_class._name, - self.method_name) def __call__(self, *args, **kwargs): raise TypeError(f"Remote method cannot be called directly. " "Use {self._name}.remote() instead") + def _get_ray_remote_impl(self): + return getattr(self.actor_handle._get_ray_remote_impl(), + self.method_name) + + def __getstate__(self) -> Dict: + state = { + "actor_handle": self.actor_handle, + "method_name": self.method_name, + } + return state + + def __setstate__(self, state: Dict) -> None: + self.actor_handle = state["actor_handle"] + self.method_name = state["method_name"] + def remote(self, *args, **kwargs): - return ray.call_remote(self, ray_client_pb2.ClientTask.METHOD, *args, - **kwargs) + return ray.call_remote(self, *args, **kwargs) def __repr__(self): - return "ClientRemoteMethod(%s, %s)" % (self._name, self.actor_id) + name = "%s.%s" % (self.actor_handle.actor_class._name, + self.method_name) + return "ClientRemoteMethod(%s, %s)" % (name, + self.actor_handle.actor_id) + + def _prepare_client_task(self) -> ray_client_pb2.ClientTask: + task = ray_client_pb2.ClientTask() + task.type = ray_client_pb2.ClientTask.METHOD + task.name = self.method_name + task.payload_id = self.actor_handle.actor_ref.handle + return task def convert_from_arg(pb) -> Any: @@ -114,3 +278,13 @@ def convert_to_arg(val): out.local = ray_client_pb2.Arg.Locality.INTERNED out.data = cloudpickle.dumps(val) return out + + +def encode_exception(exception) -> str: + data = cloudpickle.dumps(exception) + return base64.standard_b64encode(data).decode() + + +def decode_exception(data) -> Exception: + data = base64.standard_b64decode(data) + return cloudpickle.loads(data) diff --git a/python/ray/experimental/client/server/core_ray_api.py b/python/ray/experimental/client/server/core_ray_api.py index 3ebb36c32..6513021a8 100644 --- a/python/ray/experimental/client/server/core_ray_api.py +++ b/python/ray/experimental/client/server/core_ray_api.py @@ -7,18 +7,36 @@ # While the stub is trivial, it allows us to check that the calls we're # making into the core-ray module are contained and well-defined. +from typing import Any +from typing import Optional +from typing import Union + import ray from ray.experimental.client.api import APIImpl -from ray.experimental.client.common import ClientRemoteFunc +from ray.experimental.client.common import ClientObjectRef +from ray.experimental.client.common import ClientStub class CoreRayAPI(APIImpl): - def get(self, *args, **kwargs): - return ray.get(*args, **kwargs) + """ + Implements the equivalent client-side Ray API by simply passing along to + the Core Ray API. Primarily used inside of Ray Workers as a trampoline back + to core ray when passed client stubs. + """ - def put(self, *args, **kwargs): - return ray.put(*args, **kwargs) + def get(self, vals, *, timeout: Optional[float] = None) -> Any: + if isinstance(vals, list): + if isinstance(vals[0], ClientObjectRef): + return ray.get( + [val._unpack_ref() for val in vals], timeout=timeout) + elif isinstance(vals, ClientObjectRef): + return ray.get(vals._unpack_ref(), timeout=timeout) + return ray.get(vals, timeout=timeout) + + def put(self, vals: Any, *args, + **kwargs) -> Union[ClientObjectRef, ray._raylet.ObjectRef]: + return ray.put(vals, *args, **kwargs) def wait(self, *args, **kwargs): return ray.wait(*args, **kwargs) @@ -26,16 +44,58 @@ class CoreRayAPI(APIImpl): def remote(self, *args, **kwargs): return ray.remote(*args, **kwargs) - def call_remote(self, f: ClientRemoteFunc, kind: int, *args, **kwargs): - if f._raylet_remote_func is None: - f._raylet_remote_func = ray.remote(f._func) - return f._raylet_remote_func.remote(*args, **kwargs) + def call_remote(self, instance: ClientStub, *args, **kwargs): + return instance._get_ray_remote_impl().remote(*args, **kwargs) - def close(self, *args, **kwargs): + def close(self) -> None: return None + def kill(self, actor, *, no_restart=True): + return ray.kill(actor, no_restart=no_restart) + + def cancel(self, obj, *, force=False, recursive=True): + return ray.cancel(obj, force=force, recursive=recursive) + + def is_initialized(self) -> bool: + return ray.is_initialized() + # Allow for generic fallback to ray.* in remote methods. This allows calls # like ray.nodes() to be run in remote functions even though the client # doesn't currently support them. def __getattr__(self, key: str): return getattr(ray, key) + + +class RayServerAPI(CoreRayAPI): + """ + Ray Client server-side API shim. By default, simply calls the default Core + Ray API calls, but also accepts scheduling calls from functions running + inside of other remote functions that need to create more work. + """ + + def __init__(self, server_instance): + self.server = server_instance + + # Wrap single item into list if needed before calling server put. + def put(self, vals: Any, *args, **kwargs) -> ClientObjectRef: + to_put = [] + single = False + if isinstance(vals, list): + to_put = vals + else: + single = True + to_put.append(vals) + + out = [self._put(x) for x in to_put] + if single: + out = out[0] + return out + + def _put(self, val: Any): + resp = self.server._put_and_retain_obj(val) + return ClientObjectRef(resp.id) + + def call_remote(self, instance: ClientStub, *args, **kwargs): + task = instance._prepare_client_task() + ticket = self.server.Schedule(task, prepared_args=args) + return ClientObjectRef(ticket.return_id) diff --git a/python/ray/experimental/client/server/server.py b/python/ray/experimental/client/server/server.py index e42ea8db4..616e6e60d 100644 --- a/python/ray/experimental/client/server/server.py +++ b/python/ray/experimental/client/server/server.py @@ -3,14 +3,17 @@ from concurrent import futures import grpc from ray import cloudpickle import ray +import ray.state import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc import time import inspect -from ray.experimental.client import stash_api_for_tests +import json +from ray.experimental.client import stash_api_for_tests, _set_server_api from ray.experimental.client.common import convert_from_arg +from ray.experimental.client.common import encode_exception from ray.experimental.client.common import ClientObjectRef -from ray.experimental.client.common import ClientRemoteFunc +from ray.experimental.client.server.core_ray_api import RayServerAPI logger = logging.getLogger(__name__) @@ -23,31 +26,98 @@ class RayletServicer(ray_client_pb2_grpc.RayletDriverServicer): self.registered_actor_classes = {} self._test_mode = test_mode + def ClusterInfo(self, request, + context=None) -> ray_client_pb2.ClusterInfoResponse: + resp = ray_client_pb2.ClusterInfoResponse() + resp.type = request.type + if request.type == ray_client_pb2.ClusterInfoType.CLUSTER_RESOURCES: + resources = ray.cluster_resources() + # Normalize resources into floats + # (the function may return values that are ints) + float_resources = {k: float(v) for k, v in resources.items()} + resp.resource_table.CopyFrom( + ray_client_pb2.ClusterInfoResponse.ResourceTable( + table=float_resources)) + elif request.type == \ + ray_client_pb2.ClusterInfoType.AVAILABLE_RESOURCES: + resources = ray.available_resources() + # Normalize resources into floats + # (the function may return values that are ints) + float_resources = {k: float(v) for k, v in resources.items()} + resp.resource_table.CopyFrom( + ray_client_pb2.ClusterInfoResponse.ResourceTable( + table=float_resources)) + else: + resp.json = self._return_debug_cluster_info(request, context) + return resp + + def _return_debug_cluster_info(self, request, context=None) -> str: + data = None + if request.type == ray_client_pb2.ClusterInfoType.NODES: + data = ray.nodes() + elif request.type == ray_client_pb2.ClusterInfoType.IS_INITIALIZED: + data = ray.is_initialized() + else: + raise TypeError("Unsupported cluster info type") + return json.dumps(data) + + def Terminate(self, request, context=None): + if request.WhichOneof("terminate_type") == "task_object": + try: + object_ref = cloudpickle.loads(request.task_object.handle) + ray.cancel( + object_ref, + force=request.task_object.force, + recursive=request.task_object.recursive) + except Exception as e: + return_exception_in_context(e, context) + elif request.WhichOneof("terminate_type") == "actor": + try: + actor_ref = cloudpickle.loads(request.actor.handle) + ray.kill(actor_ref, no_restart=request.actor.no_restart) + except Exception as e: + return_exception_in_context(e, context) + else: + raise RuntimeError( + "Client requested termination without providing a valid " + "terminate_type") + return ray_client_pb2.TerminateResponse(ok=True) + def GetObject(self, request, context=None): - if request.id not in self.object_refs: + request_ref = cloudpickle.loads(request.handle) + if request_ref.binary() not in self.object_refs: return ray_client_pb2.GetResponse(valid=False) - objectref = self.object_refs[request.id] + objectref = self.object_refs[request_ref.binary()] logger.info("get: %s" % objectref) - item = ray.get(objectref) + try: + item = ray.get(objectref, timeout=request.timeout) + except Exception as e: + return_exception_in_context(e, context) item_ser = cloudpickle.dumps(item) return ray_client_pb2.GetResponse(valid=True, data=item_ser) - def PutObject(self, request, context=None): + def PutObject(self, request, context=None) -> ray_client_pb2.PutResponse: obj = cloudpickle.loads(request.data) + objectref = self._put_and_retain_obj(obj) + pickled_ref = cloudpickle.dumps(objectref) + return ray_client_pb2.PutResponse( + ref=make_remote_ref(objectref.binary(), pickled_ref)) + + def _put_and_retain_obj(self, obj) -> ray.ObjectRef: objectref = ray.put(obj) self.object_refs[objectref.binary()] = objectref logger.info("put: %s" % objectref) - return ray_client_pb2.PutResponse(id=objectref.binary()) + return objectref def WaitObject(self, request, context=None) -> ray_client_pb2.WaitResponse: - object_refs = [cloudpickle.loads(o) for o in request.object_refs] + object_refs = [cloudpickle.loads(o) for o in request.object_handles] num_returns = request.num_returns timeout = request.timeout object_refs_ids = [] for object_ref in object_refs: - if object_ref.id not in self.object_refs: + if object_ref.binary() not in self.object_refs: return ray_client_pb2.WaitResponse(valid=False) - object_refs_ids.append(self.object_refs[object_ref.id]) + object_refs_ids.append(self.object_refs[object_ref.binary()]) try: ready_object_refs, remaining_object_refs = ray.wait( object_refs_ids, @@ -59,94 +129,133 @@ class RayletServicer(ray_client_pb2_grpc.RayletDriverServicer): logger.info("wait: %s %s" % (str(ready_object_refs), str(remaining_object_refs))) ready_object_ids = [ - ready_object_ref.binary() for ready_object_ref in ready_object_refs + make_remote_ref( + id=ready_object_ref.binary(), + handle=cloudpickle.dumps(ready_object_ref), + ) for ready_object_ref in ready_object_refs ] remaining_object_ids = [ - remaining_object_ref.binary() - for remaining_object_ref in remaining_object_refs + make_remote_ref( + id=remaining_object_ref.binary(), + handle=cloudpickle.dumps(remaining_object_ref), + ) for remaining_object_ref in remaining_object_refs ] return ray_client_pb2.WaitResponse( valid=True, ready_object_ids=ready_object_ids, remaining_object_ids=remaining_object_ids) - def Schedule(self, task, context=None) -> ray_client_pb2.ClientTaskTicket: + def Schedule(self, task, context=None, + prepared_args=None) -> ray_client_pb2.ClientTaskTicket: logger.info("schedule: %s %s" % (task.name, ray_client_pb2.ClientTask.RemoteExecType.Name(task.type))) if task.type == ray_client_pb2.ClientTask.FUNCTION: - return self._schedule_function(task, context) + return self._schedule_function(task, context, prepared_args) elif task.type == ray_client_pb2.ClientTask.ACTOR: - return self._schedule_actor(task, context) + return self._schedule_actor(task, context, prepared_args) elif task.type == ray_client_pb2.ClientTask.METHOD: - return self._schedule_method(task, context) + return self._schedule_method(task, context, prepared_args) else: raise NotImplementedError( "Unimplemented Schedule task type: %s" % ray_client_pb2.ClientTask.RemoteExecType.Name(task.type)) - def _schedule_method(self, task: ray_client_pb2.ClientTask, - context=None) -> ray_client_pb2.ClientTaskTicket: + def _schedule_method( + self, + task: ray_client_pb2.ClientTask, + context=None, + prepared_args=None) -> ray_client_pb2.ClientTaskTicket: actor_handle = self.actor_refs.get(task.payload_id) if actor_handle is None: raise Exception( "Can't run an actor the server doesn't have a handle for") - arglist = _convert_args(task.args) + arglist = _convert_args(task.args, prepared_args) with stash_api_for_tests(self._test_mode): output = getattr(actor_handle, task.name).remote(*arglist) self.object_refs[output.binary()] = output - return ray_client_pb2.ClientTaskTicket(return_id=output.binary()) + pickled_ref = cloudpickle.dumps(output) + return ray_client_pb2.ClientTaskTicket( + return_ref=make_remote_ref(output.binary(), pickled_ref)) - def _schedule_actor(self, task: ray_client_pb2.ClientTask, - context=None) -> ray_client_pb2.ClientTaskTicket: + def _schedule_actor(self, + task: ray_client_pb2.ClientTask, + context=None, + prepared_args=None) -> ray_client_pb2.ClientTaskTicket: with stash_api_for_tests(self._test_mode): - if task.payload_id not in self.registered_actor_classes: - actor_class_ref = self.object_refs[task.payload_id] + payload_ref = cloudpickle.loads(task.payload_id) + if payload_ref.binary() not in self.registered_actor_classes: + actor_class_ref = self.object_refs[payload_ref.binary()] actor_class = ray.get(actor_class_ref) if not inspect.isclass(actor_class): raise Exception("Attempting to schedule actor that " - "isn't a ClientActorClass.") + "isn't a class.") reg_class = ray.remote(actor_class) - self.registered_actor_classes[task.payload_id] = reg_class - remote_class = self.registered_actor_classes[task.payload_id] - arglist = _convert_args(task.args) + self.registered_actor_classes[payload_ref.binary()] = reg_class + remote_class = self.registered_actor_classes[payload_ref.binary()] + arglist = _convert_args(task.args, prepared_args) actor = remote_class.remote(*arglist) - actor_ref = actor._actor_id - self.actor_refs[actor_ref.binary()] = actor - return ray_client_pb2.ClientTaskTicket(return_id=actor_ref.binary()) + actorhandle = cloudpickle.dumps(actor) + self.actor_refs[actorhandle] = actor + return ray_client_pb2.ClientTaskTicket( + return_ref=make_remote_ref(actor._actor_id.binary(), actorhandle)) - def _schedule_function(self, task: ray_client_pb2.ClientTask, - context=None) -> ray_client_pb2.ClientTaskTicket: - if task.payload_id not in self.function_refs: - funcref = self.object_refs[task.payload_id] + def _schedule_function( + self, + task: ray_client_pb2.ClientTask, + context=None, + prepared_args=None) -> ray_client_pb2.ClientTaskTicket: + payload_ref = cloudpickle.loads(task.payload_id) + if payload_ref.binary() not in self.function_refs: + funcref = self.object_refs[payload_ref.binary()] func = ray.get(funcref) - if not isinstance(func, ClientRemoteFunc): + if not inspect.isfunction(func): raise Exception("Attempting to schedule function that " - "isn't a ClientRemoteFunc.") - self.function_refs[task.payload_id] = func - remote_func = self.function_refs[task.payload_id] - arglist = _convert_args(task.args) + "isn't a function.") + self.function_refs[payload_ref.binary()] = ray.remote(func) + remote_func = self.function_refs[payload_ref.binary()] + arglist = _convert_args(task.args, prepared_args) # Prepare call if we're in a test with stash_api_for_tests(self._test_mode): output = remote_func.remote(*arglist) + if output.binary() in self.object_refs: + raise Exception("already found it") self.object_refs[output.binary()] = output - return ray_client_pb2.ClientTaskTicket(return_id=output.binary()) + pickled_output = cloudpickle.dumps(output) + return ray_client_pb2.ClientTaskTicket( + return_ref=make_remote_ref(output.binary(), pickled_output)) -def _convert_args(arg_list): +def _convert_args(arg_list, prepared_args=None): + if prepared_args is not None: + return prepared_args out = [] for arg in arg_list: t = convert_from_arg(arg) if isinstance(t, ClientObjectRef): - out.append(ray.ObjectRef(t.id)) + out.append(t._unpack_ref()) else: out.append(t) return out +def make_remote_ref(id: bytes, handle: bytes) -> ray_client_pb2.RemoteRef: + return ray_client_pb2.RemoteRef( + id=id, + handle=handle, + ) + + +def return_exception_in_context(err, context): + if context is not None: + context.set_details(encode_exception(err)) + context.set_code(grpc.StatusCode.INTERNAL) + + def serve(connection_str, test_mode=False): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) task_servicer = RayletServicer(test_mode=test_mode) + _set_server_api(RayServerAPI(task_servicer)) ray_client_pb2_grpc.add_RayletDriverServicer_to_server( task_servicer, server) server.add_insecure_port(connection_str) diff --git a/python/ray/experimental/client/worker.py b/python/ray/experimental/client/worker.py index 8c01bea34..0a108e4f2 100644 --- a/python/ray/experimental/client/worker.py +++ b/python/ray/experimental/client/worker.py @@ -3,22 +3,29 @@ It implements the Ray API functions that are forwarded through grpc calls to the server. """ import inspect +import json +import logging +from typing import Any from typing import List from typing import Tuple +from typing import Optional import ray.cloudpickle as cloudpickle from ray.util.inspect import is_cython import grpc +from ray.exceptions import TaskCancelledError import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc from ray.experimental.client.common import convert_to_arg +from ray.experimental.client.common import decode_exception from ray.experimental.client.common import ClientObjectRef -from ray.experimental.client.common import ClientActorRef from ray.experimental.client.common import ClientActorClass -from ray.experimental.client.common import ClientRemoteMethod +from ray.experimental.client.common import ClientActorHandle from ray.experimental.client.common import ClientRemoteFunc +logger = logging.getLogger(__name__) + class Worker: def __init__(self, @@ -34,6 +41,7 @@ class Worker: metadata: additional metadata passed in the grpc request headers. """ self.metadata = metadata + self.channel = None if stub is None: if secure: credentials = grpc.ssl_channel_credentials() @@ -44,28 +52,32 @@ class Worker: else: self.server = stub - def get(self, ids): + def get(self, vals, *, timeout: Optional[float] = None) -> Any: to_get = [] single = False - if isinstance(ids, list): - to_get = [x.id for x in ids] - elif isinstance(ids, ClientObjectRef): - to_get = [ids.id] + if isinstance(vals, list): + to_get = [x.handle for x in vals] + elif isinstance(vals, ClientObjectRef): + to_get = [vals.handle] single = True else: raise Exception("Can't get something that's not a " - "list of IDs or just an ID: %s" % type(ids)) - out = [self._get(x) for x in to_get] + "list of IDs or just an ID: %s" % type(vals)) + if timeout is None: + timeout = 0 + out = [self._get(x, timeout) for x in to_get] if single: out = out[0] return out - def _get(self, id: bytes): - req = ray_client_pb2.GetRequest(id=id) - data = self.server.GetObject(req, metadata=self.metadata) + def _get(self, handle: bytes, timeout: float): + req = ray_client_pb2.GetRequest(handle=handle, timeout=timeout) + try: + data = self.server.GetObject(req, metadata=self.metadata) + except grpc.RpcError as e: + raise decode_exception(e.details()) if not data.valid: - raise Exception( - "Client GetObject returned invalid data: id invalid?") + raise TaskCancelledError(handle) return cloudpickle.loads(data.data) def put(self, vals): @@ -86,7 +98,7 @@ class Worker: data = cloudpickle.dumps(val) req = ray_client_pb2.PutRequest(data=data) resp = self.server.PutObject(req, metadata=self.metadata) - return ClientObjectRef(resp.id) + return ClientObjectRef.from_remote_ref(resp.ref) def wait(self, object_refs: List[ClientObjectRef], @@ -98,8 +110,8 @@ class Worker: for ref in object_refs: assert isinstance(ref, ClientObjectRef) data = { - "object_refs": [ - cloudpickle.dumps(object_ref) for object_ref in object_refs + "object_handles": [ + object_ref.handle for object_ref in object_refs ], "num_returns": num_returns, "timeout": timeout if timeout else -1 @@ -110,10 +122,12 @@ class Worker: # TODO(ameer): improve error/exceptions messages. raise Exception("Client Wait request failed. Reference invalid?") client_ready_object_ids = [ - ClientObjectRef(id) for id in resp.ready_object_ids + ClientObjectRef.from_remote_ref(ref) + for ref in resp.ready_object_ids ] client_remaining_object_ids = [ - ClientObjectRef(id) for id in resp.remaining_object_ids + ClientObjectRef.from_remote_ref(ref) + for ref in resp.remaining_object_ids ] return (client_ready_object_ids, client_remaining_object_ids) @@ -130,50 +144,60 @@ class Worker: raise TypeError("The @ray.remote decorator must be applied to " "either a function or to a class.") - def call_remote(self, instance, kind, *args, **kwargs): - ticket = None - if kind == ray_client_pb2.ClientTask.FUNCTION: - ticket = self._put_and_schedule(instance, kind, *args, **kwargs) - elif kind == ray_client_pb2.ClientTask.ACTOR: - ticket = self._put_and_schedule(instance, kind, *args, **kwargs) - return ClientActorRef(ticket.return_id) - elif kind == ray_client_pb2.ClientTask.METHOD: - ticket = self._call_method(instance, *args, **kwargs) - - if ticket is None: - raise Exception( - "Couldn't call_remote on %s for type %s" % (instance, kind)) - return ClientObjectRef(ticket.return_id) - - def _call_method(self, instance: ClientRemoteMethod, *args, **kwargs): - if not isinstance(instance, ClientRemoteMethod): - raise TypeError("Client not passing a ClientRemoteMethod stub") - task = ray_client_pb2.ClientTask() - task.type = ray_client_pb2.ClientTask.METHOD - task.name = instance.method_name - task.payload_id = instance.actor_handle.actor_id.id + def call_remote(self, instance, *args, **kwargs): + task = instance._prepare_client_task() for arg in args: pb_arg = convert_to_arg(arg) task.args.append(pb_arg) + logging.debug("Scheduling %s" % task) ticket = self.server.Schedule(task, metadata=self.metadata) - return ticket - - def _put_and_schedule(self, item, task_type, *args, **kwargs): - if isinstance(item, ClientRemoteFunc): - ref = self._put(item) - elif isinstance(item, ClientActorClass): - ref = self._put(item.actor_cls) - else: - raise TypeError("Client not passing a ClientRemoteFunc stub") - task = ray_client_pb2.ClientTask() - task.type = task_type - task.name = item._name - task.payload_id = ref.id - for arg in args: - pb_arg = convert_to_arg(arg) - task.args.append(pb_arg) - ticket = self.server.Schedule(task, metadata=self.metadata) - return ticket + return ClientObjectRef.from_remote_ref(ticket.return_ref) def close(self): - self.channel.close() + self.server = None + if self.channel: + self.channel.close() + + def terminate_actor(self, actor: ClientActorHandle, + no_restart: bool) -> None: + if not isinstance(actor, ClientActorHandle): + raise ValueError("ray.kill() only supported for actors. " + "Got: {}.".format(type(actor))) + term_actor = ray_client_pb2.TerminateRequest.ActorTerminate() + term_actor.handle = actor.actor_ref.handle + term_actor.no_restart = no_restart + try: + term = ray_client_pb2.TerminateRequest(actor=term_actor) + self.server.Terminate(term) + except grpc.RpcError as e: + raise decode_exception(e.details()) + + def terminate_task(self, obj: ClientObjectRef, force: bool, + recursive: bool) -> None: + if not isinstance(obj, ClientObjectRef): + raise TypeError( + "ray.cancel() only supported for non-actor object refs. " + f"Got: {type(obj)}.") + term_object = ray_client_pb2.TerminateRequest.TaskObjectTerminate() + term_object.handle = obj.handle + term_object.force = force + term_object.recursive = recursive + try: + term = ray_client_pb2.TerminateRequest(task_object=term_object) + self.server.Terminate(term) + except grpc.RpcError as e: + raise decode_exception(e.details()) + + def get_cluster_info(self, type: ray_client_pb2.ClusterInfoType.TypeEnum): + req = ray_client_pb2.ClusterInfoRequest() + req.type = type + resp = self.server.ClusterInfo(req) + if resp.WhichOneof("response_type") == "resource_table": + return resp.resource_table.table + return json.loads(resp.json) + + def is_initialized(self) -> bool: + if self.server is not None: + return self.get_cluster_info( + ray_client_pb2.ClusterInfoType.IS_INITIALIZED) + return False diff --git a/python/ray/gcs_utils.py b/python/ray/gcs_utils.py index ef95b4c0a..2aed4526d 100644 --- a/python/ray/gcs_utils.py +++ b/python/ray/gcs_utils.py @@ -7,8 +7,8 @@ from ray.core.generated.gcs_pb2 import ( JobConfig, ErrorTableData, GcsEntry, - HeartbeatBatchTableData, - HeartbeatTableData, + ResourceUsageBatchData, + ResourcesData, ObjectTableData, ProfileTableData, TablePrefix, @@ -33,8 +33,8 @@ __all__ = [ "ErrorTableData", "ErrorType", "GcsEntry", - "HeartbeatBatchTableData", - "HeartbeatTableData", + "ResourceUsageBatchData", + "ResourcesData", "ObjectTableData", "ProfileTableData", "TablePrefix", @@ -55,8 +55,8 @@ FUNCTION_PREFIX = "RemoteFunction:" LOG_FILE_CHANNEL = "RAY_LOG_CHANNEL" REPORTER_CHANNEL = "RAY_REPORTER" -# xray heartbeats -XRAY_HEARTBEAT_BATCH_PATTERN = "HEARTBEAT_BATCH:".encode("ascii") +# xray resource usages +XRAY_RESOURCES_BATCH_PATTERN = "RESOURCES_BATCH:".encode("ascii") # xray job updates XRAY_JOB_PATTERN = "JOB:*".encode("ascii") diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index 2d0c70c32..31418f10c 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -23,7 +23,7 @@ cdef extern from "ray/gcs/gcs_client/global_state_accessor.h" nogil: c_vector[c_string] GetAllProfileInfo() c_vector[c_string] GetAllObjectInfo() unique_ptr[c_string] GetObjectInfo(const CObjectID &object_id) - unique_ptr[c_string] GetAllHeartbeat() + unique_ptr[c_string] GetAllResourceUsage() c_vector[c_string] GetAllActorInfo() unique_ptr[c_string] GetActorInfo(const CActorID &actor_id) c_string GetNodeResourceInfo(const CNodeID &node_id) diff --git a/python/ray/includes/global_state_accessor.pxi b/python/ray/includes/global_state_accessor.pxi index 25a88028b..cbb1bac0a 100644 --- a/python/ray/includes/global_state_accessor.pxi +++ b/python/ray/includes/global_state_accessor.pxi @@ -78,11 +78,11 @@ cdef class GlobalStateAccessor: return c_string(object_info.get().data(), object_info.get().size()) return None - def get_all_heartbeat(self): - """Get newest heartbeat of all nodes from GCS service.""" + def get_all_resource_usage(self): + """Get newest resource usage of all nodes from GCS service.""" cdef unique_ptr[c_string] result with nogil: - result = self.inner.get().GetAllHeartbeat() + result = self.inner.get().GetAllResourceUsage() if result: return c_string(result.get().data(), result.get().size()) return None diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index c7647ad49..abf1290b9 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -90,7 +90,8 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: const CTaskOptions &options, c_vector[CObjectID] *return_ids, int max_retries, c_pair[CPlacementGroupID, int64_t] placement_options, - c_bool placement_group_capture_child_tasks) + c_bool placement_group_capture_child_tasks, + c_string debugger_breakpoint) CRayStatus CreateActor( const CRayFunction &function, const c_vector[unique_ptr[CTaskArg]] &args, @@ -101,6 +102,8 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: CPlacementGroupID *placement_group_id) CRayStatus RemovePlacementGroup( const CPlacementGroupID &placement_group_id) + CRayStatus WaitPlacementGroupReady( + const CPlacementGroupID &placement_group_id, int timeout_ms) void SubmitActorTask( const CActorID &actor_id, const CRayFunction &function, const c_vector[unique_ptr[CTaskArg]] &args, @@ -222,6 +225,7 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: const c_vector[shared_ptr[CRayObject]] &args, const c_vector[CObjectID] &arg_reference_ids, const c_vector[CObjectID] &return_ids, + const c_string debugger_breakpoint, c_vector[shared_ptr[CRayObject]] *returns) nogil ) task_execution_callback (void(const CWorkerID &) nogil) on_worker_shutdown diff --git a/python/ray/includes/ray_config.pxd b/python/ray/includes/ray_config.pxd index 1a1122136..03979863d 100644 --- a/python/ray/includes/ray_config.pxd +++ b/python/ray/includes/ray_config.pxd @@ -15,7 +15,7 @@ cdef extern from "ray/common/ray_config.h" nogil: int64_t raylet_heartbeat_timeout_milliseconds() const - c_bool light_heartbeat_enabled() const + c_bool light_report_resource_usage_enabled() const int64_t debug_dump_period_milliseconds() const @@ -51,10 +51,6 @@ cdef extern from "ray/common/ray_config.h" nogil: uint64_t object_manager_default_chunk_size() const - int num_workers_per_process_python() const - - int num_workers_per_process_java() const - uint32_t maximum_gcs_deletion_batch_size() const int64_t max_direct_call_object_size() const @@ -68,3 +64,5 @@ cdef extern from "ray/common/ray_config.h" nogil: c_bool enable_timeline() const c_bool automatic_object_deletion_enabled() const + + uint32_t max_grpc_message_size() const diff --git a/python/ray/includes/ray_config.pxi b/python/ray/includes/ray_config.pxi index 7db833cbc..b9e26cf08 100644 --- a/python/ray/includes/ray_config.pxi +++ b/python/ray/includes/ray_config.pxi @@ -14,8 +14,8 @@ cdef class Config: return RayConfig.instance().raylet_heartbeat_timeout_milliseconds() @staticmethod - def light_heartbeat_enabled(): - return RayConfig.instance().light_heartbeat_enabled() + def light_report_resource_usage_enabled(): + return RayConfig.instance().light_report_resource_usage_enabled() @staticmethod def debug_dump_period_milliseconds(): @@ -88,14 +88,6 @@ cdef class Config: def object_manager_default_chunk_size(): return RayConfig.instance().object_manager_default_chunk_size() - @staticmethod - def num_workers_per_process_python(): - return RayConfig.instance().num_workers_per_process_python() - - @staticmethod - def num_workers_per_process_java(): - return RayConfig.instance().num_workers_per_process_java() - @staticmethod def maximum_gcs_deletion_batch_size(): return RayConfig.instance().maximum_gcs_deletion_batch_size() @@ -119,3 +111,7 @@ cdef class Config: @staticmethod def automatic_object_deletion_enabled(): return RayConfig.instance().automatic_object_deletion_enabled() + + @staticmethod + def max_grpc_message_size(): + return RayConfig.instance().max_grpc_message_size() diff --git a/python/ray/internal/internal_api.py b/python/ray/internal/internal_api.py index 04c260ffa..d3e25c1ec 100644 --- a/python/ray/internal/internal_api.py +++ b/python/ray/internal/internal_api.py @@ -1,7 +1,9 @@ +import ray import ray.worker from ray import profiling __all__ = ["free", "global_gc"] +MAX_MESSAGE_LENGTH = ray._config.max_grpc_message_size() def global_gc(): @@ -22,7 +24,13 @@ def memory_summary(): raylet = ray.nodes()[0] raylet_address = "{}:{}".format(raylet["NodeManagerAddress"], ray.nodes()[0]["NodeManagerPort"]) - channel = grpc.insecure_channel(raylet_address) + channel = grpc.insecure_channel( + raylet_address, + options=[ + ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH), + ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH), + ], + ) stub = node_manager_pb2_grpc.NodeManagerServiceStub(channel) reply = stub.FormatGlobalMemoryInfo( node_manager_pb2.FormatGlobalMemoryInfoRequest(), timeout=30.0) diff --git a/python/ray/log_monitor.py b/python/ray/log_monitor.py index 4932b7338..ac5fa5296 100644 --- a/python/ray/log_monitor.py +++ b/python/ray/log_monitor.py @@ -133,7 +133,7 @@ class LogMonitor: job_match = JOB_LOG_PATTERN.match(file_path) if job_match: job_id = job_match.group(2) - worker_pid = job_match.group(3) + worker_pid = int(job_match.group(3)) else: job_id = None worker_pid = None @@ -361,4 +361,5 @@ if __name__ == "__main__": f"failed with the following error:\n{traceback_str}") ray.utils.push_error_to_driver_through_redis( redis_client, ray_constants.LOG_MONITOR_DIED_ERROR, message) + logger.error(message) raise e diff --git a/python/ray/memory_monitor.py b/python/ray/memory_monitor.py index 74bfee681..9381c5064 100644 --- a/python/ray/memory_monitor.py +++ b/python/ray/memory_monitor.py @@ -91,7 +91,7 @@ class MemoryMonitor: if not psutil: logger.warn("WARNING: Not monitoring node memory since `psutil` " "is not installed. Install this with " - "`pip install psutil` (or ray[debug]) to enable " + "`pip install psutil` to enable " "debugging of memory-related crashes.") def get_memory_usage(self): diff --git a/python/ray/monitor.py b/python/ray/monitor.py index 5f93e535a..f650e151a 100644 --- a/python/ray/monitor.py +++ b/python/ray/monitor.py @@ -85,7 +85,11 @@ class Monitor: This is used to receive notifications about failed components. """ - def __init__(self, redis_address, autoscaling_config, redis_password=None): + def __init__(self, + redis_address, + autoscaling_config, + redis_password=None, + prefix_cluster_info=False): # Initialize the Redis clients. ray.state.state._initialize_global_state( redis_address, redis_password=redis_password) @@ -107,8 +111,10 @@ class Monitor: head_node_ip = redis_address.split(":")[0] self.load_metrics = LoadMetrics(local_ip=head_node_ip) if autoscaling_config: - self.autoscaler = StandardAutoscaler(autoscaling_config, - self.load_metrics) + self.autoscaler = StandardAutoscaler( + autoscaling_config, + self.load_metrics, + prefix_cluster_info=prefix_cluster_info) self.autoscaling_config = autoscaling_config else: self.autoscaler = None @@ -139,24 +145,24 @@ class Monitor: self.primary_subscribe_client.subscribe(channel) def update_load_metrics(self): - """Fetches heartbeat data from GCS and updates load metrics.""" + """Fetches resource usage data from GCS and updates load metrics.""" - all_heartbeat = self.global_state_accessor.get_all_heartbeat() - heartbeat_batch_data = \ - ray.gcs_utils.HeartbeatBatchTableData.FromString(all_heartbeat) - for heartbeat_message in heartbeat_batch_data.batch: - resource_load = dict(heartbeat_message.resource_load) - total_resources = dict(heartbeat_message.resources_total) - available_resources = dict(heartbeat_message.resources_available) + all_resources = self.global_state_accessor.get_all_resource_usage() + resources_batch_data = \ + ray.gcs_utils.ResourceUsageBatchData.FromString(all_resources) + for resource_message in resources_batch_data.batch: + resource_load = dict(resource_message.resource_load) + total_resources = dict(resource_message.resources_total) + available_resources = dict(resource_message.resources_available) waiting_bundles, infeasible_bundles = parse_resource_demands( - heartbeat_batch_data.resource_load_by_shape) + resources_batch_data.resource_load_by_shape) pending_placement_groups = list( - heartbeat_batch_data.placement_group_load.placement_group_data) + resources_batch_data.placement_group_load.placement_group_data) # Update the load metrics for this raylet. - node_id = ray.utils.binary_to_hex(heartbeat_message.node_id) + node_id = ray.utils.binary_to_hex(resource_message.node_id) ip = self.raylet_id_to_ip_map.get(node_id) if ip: self.load_metrics.update(ip, total_resources, diff --git a/python/ray/nightly-wheels.yaml b/python/ray/nightly-wheels.yaml index f2d52651a..24ec338bc 100644 --- a/python/ray/nightly-wheels.yaml +++ b/python/ray/nightly-wheels.yaml @@ -1,14 +1,14 @@ linux: - "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl - "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl - "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl + "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl + "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl darwin: - "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl - "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl - "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-macosx_10_13_intel.whl + "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl + "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-macosx_10_13_intel.whl + "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-macosx_10_13_intel.whl win32: - "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-win_amd64.whl - "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-win_amd64.whl - "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-win_amd64.whl + "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-win_amd64.whl + "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-win_amd64.whl + "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-win_amd64.whl diff --git a/python/ray/node.py b/python/ray/node.py index 13d73956c..a7ec72e7a 100644 --- a/python/ray/node.py +++ b/python/ray/node.py @@ -339,10 +339,6 @@ class Node: """Get the cluster Redis password""" return self._ray_params.redis_password - @property - def load_code_from_local(self): - return self._ray_params.load_code_from_local - @property def object_ref_seed(self): """Get the seed for deterministic generation of object refs""" @@ -723,14 +719,12 @@ class Node: stderr_file=stderr_file, config=self._config, java_worker_options=self._ray_params.java_worker_options, - load_code_from_local=self._ray_params.load_code_from_local, huge_pages=self._ray_params.huge_pages, fate_share=self.kernel_fate_share, socket_to_use=self.socket, head_node=self.head, start_initial_python_workers_for_first_job=self._ray_params. - start_initial_python_workers_for_first_job, - code_search_path=self._ray_params.code_search_path) + start_initial_python_workers_for_first_job) assert ray_constants.PROCESS_TYPE_RAYLET not in self.all_processes self.all_processes[ray_constants.PROCESS_TYPE_RAYLET] = [process_info] @@ -739,12 +733,19 @@ class Node: raise NotImplementedError def start_monitor(self): - """Start the monitor.""" + """Start the monitor. + + Autoscaling output goes to these monitor.err/out files, and + any modification to these files may break existing + cluster launching commands. + """ + stdout_file, stderr_file = self.get_log_file_handles( + "monitor", unique=True) process_info = ray._private.services.start_monitor( self._redis_address, self._logs_dir, - stdout_file=subprocess.DEVNULL, - stderr_file=subprocess.DEVNULL, + stdout_file=stdout_file, + stderr_file=stderr_file, autoscaling_config=self._ray_params.autoscaling_config, redis_password=self._ray_params.redis_password, fate_share=self.kernel_fate_share) diff --git a/python/ray/operator.py b/python/ray/operator.py deleted file mode 100644 index a277b74bb..000000000 --- a/python/ray/operator.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Ray operator for Kubernetes. - -Reads ray cluster config from a k8s ConfigMap, starts a ray head node pod using -create_or_update_cluster(), then runs an autoscaling loop in the operator pod -executing this script. Writes autoscaling logs to the directory -/root/ray-operator-logs. - -In this setup, the ray head node does not run an autoscaler. It is important -NOT to supply an --autoscaling-config argument to head node's ray start command -in the cluster config when using this operator. - -To run, first create a ConfigMap named ray-operator-configmap from a ray -cluster config. Then apply the manifest at python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml - -For example: -kubectl create namespace raytest -kubectl -n raytest create configmap ray-operator-configmap --from-file=python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml -kubectl -n raytest apply -f python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml -""" # noqa -import os -from typing import Any, Dict, IO, Tuple - -import kubernetes -import yaml - -from ray._private import services -from ray.autoscaler._private.commands import create_or_update_cluster -from ray.autoscaler._private.kubernetes import core_api -from ray.utils import open_log -from ray import ray_constants - -RAY_CLUSTER_NAMESPACE = os.environ.get("RAY_OPERATOR_POD_NAMESPACE") -RAY_CONFIG_MAP = "ray-operator-configmap" -RAY_CONFIG_DIR = "/root" - -LOG_DIR = "/root/ray-operator-logs" -ERR_NAME, OUT_NAME = "ray-operator.err", "ray-operator.out" - - -def prepare_ray_cluster_config() -> str: - config_map = core_api().read_namespaced_config_map( - name=RAY_CONFIG_MAP, namespace=RAY_CLUSTER_NAMESPACE) - - # config_map.data consists of a single key:value pair - for config_file_name, config_string in config_map.data.items(): - config = yaml.safe_load(config_string) - config["provider"]["namespace"] = RAY_CLUSTER_NAMESPACE - cluster_config_path = os.path.join(RAY_CONFIG_DIR, config_file_name) - with open(cluster_config_path, "w") as file: - yaml.dump(config, file) - - return cluster_config_path - - -def get_ray_head_pod_ip(config: Dict[str, Any]) -> str: - cluster_name = config["cluster_name"] - label_selector = f"component=ray-head,ray-cluster-name={cluster_name}" - pods = core_api().list_namespaced_pod( - namespace=RAY_CLUSTER_NAMESPACE, label_selector=label_selector).items - assert (len(pods)) == 1 - head_pod = pods.pop() - return head_pod.status.pod_ip - - -def get_logs() -> Tuple[IO, IO]: - try: - os.makedirs(LOG_DIR) - except OSError: - pass - - err_path = os.path.join(LOG_DIR, ERR_NAME) - out_path = os.path.join(LOG_DIR, OUT_NAME) - - return open_log(err_path), open_log(out_path) - - -def main(): - kubernetes.config.load_incluster_config() - cluster_config_path = prepare_ray_cluster_config() - - config = create_or_update_cluster( - cluster_config_path, - override_min_workers=None, - override_max_workers=None, - no_restart=False, - restart_only=False, - yes=True, - no_config_cache=True) - with open(cluster_config_path, "w") as file: - yaml.dump(config, file) - - ray_head_pod_ip = get_ray_head_pod_ip(config) - # TODO: Add support for user-specified redis port and password - redis_address = services.address(ray_head_pod_ip, - ray_constants.DEFAULT_PORT) - stderr_file, stdout_file = get_logs() - - services.start_monitor( - redis_address, - stdout_file=stdout_file, - stderr_file=stderr_file, - autoscaling_config=cluster_config_path, - redis_password=ray_constants.REDIS_DEFAULT_PASSWORD) - - -if __name__ == "__main__": - main() diff --git a/python/ray/operator/__init__.py b/python/ray/operator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/ray/operator/operator.py b/python/ray/operator/operator.py new file mode 100644 index 000000000..9771810c1 --- /dev/null +++ b/python/ray/operator/operator.py @@ -0,0 +1,154 @@ +""" +Ray operator for Kubernetes. + +Reads ray cluster config from a k8s ConfigMap, starts a ray head node pod using +create_or_update_cluster(), then runs an autoscaling loop in the operator pod +executing this script. Writes autoscaling logs to the directory +/root/ray-operator-logs. + +In this setup, the ray head node does not run an autoscaler. It is important +NOT to supply an --autoscaling-config argument to head node's ray start command +in the cluster config when using this operator. + +To run, first create a ConfigMap named ray-operator-configmap from a ray +cluster config. Then apply the manifest at python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml + +For example: +kubectl create namespace raytest +kubectl -n raytest create configmap ray-operator-configmap --from-file=python/ray/autoscaler/kubernetes/operator_configs/test_cluster_config.yaml +kubectl -n raytest apply -f python/ray/autoscaler/kubernetes/operator_configs/operator_config.yaml +""" # noqa +import logging +import multiprocessing as mp +import os +from typing import Any, Callable, Dict, Optional + +from kubernetes.client.exceptions import ApiException +import yaml + +from ray._private import services +from ray.autoscaler._private import commands +from ray import monitor +from ray.operator import operator_utils +from ray import ray_constants + + +class RayCluster(): + def __init__(self, config: Dict[str, Any]): + self.config = config + self.name = self.config["cluster_name"] + self.config_path = operator_utils.config_path(self.name) + + self.setup_logging() + + self.subprocess = None # type: Optional[mp.Process] + + def do_in_subprocess(self, + f: Callable[[], None], + wait_to_finish: bool = False) -> None: + # First stop the subprocess if it's alive + self.clean_up_subprocess() + # Reinstantiate process with f as target and start. + self.subprocess = mp.Process(name=self.name, target=f) + # Kill subprocess if monitor dies + self.subprocess.daemon = True + self.subprocess.start() + if wait_to_finish: + self.subprocess.join() + + def clean_up_subprocess(self): + if self.subprocess and self.subprocess.is_alive(): + self.subprocess.terminate() + self.subprocess.join() + + def create_or_update(self) -> None: + self.do_in_subprocess(self._create_or_update) + + def _create_or_update(self) -> None: + self.start_head() + self.start_monitor() + + def start_head(self) -> None: + self.write_config() + self.config = commands.create_or_update_cluster( + self.config_path, + override_min_workers=None, + override_max_workers=None, + no_restart=False, + restart_only=False, + yes=True, + no_config_cache=True) + self.write_config() + + def start_monitor(self) -> None: + ray_head_pod_ip = commands.get_head_node_ip(self.config_path) + # TODO: Add support for user-specified redis port and password + redis_address = services.address(ray_head_pod_ip, + ray_constants.DEFAULT_PORT) + self.mtr = monitor.Monitor( + redis_address=redis_address, + autoscaling_config=self.config_path, + redis_password=ray_constants.REDIS_DEFAULT_PASSWORD, + prefix_cluster_info=True) + self.mtr.run() + + def clean_up(self) -> None: + self.clean_up_subprocess() + self.clean_up_logging() + self.delete_config() + + def setup_logging(self) -> None: + self.handler = logging.StreamHandler() + self.handler.addFilter(lambda rec: rec.processName == self.name) + logging_format = ":".join([self.name, ray_constants.LOGGER_FORMAT]) + self.handler.setFormatter(logging.Formatter(logging_format)) + operator_utils.root_logger.addHandler(self.handler) + + def clean_up_logging(self) -> None: + operator_utils.root_logger.removeHandler(self.handler) + + def write_config(self) -> None: + with open(self.config_path, "w") as file: + yaml.dump(self.config, file) + + def delete_config(self) -> None: + os.remove(self.config_path) + + +ray_clusters = {} + + +def cluster_action(cluster_config: Dict[str, Any], event_type: str) -> None: + cluster_name = cluster_config["cluster_name"] + if event_type == "ADDED": + ray_clusters[cluster_name] = RayCluster(cluster_config) + ray_clusters[cluster_name].create_or_update() + elif event_type == "MODIFIED": + ray_clusters[cluster_name].create_or_update() + elif event_type == "DELETED": + ray_clusters[cluster_name].clean_up() + del ray_clusters[cluster_name] + + +def main() -> None: + # Make directory for ray cluster configs + if not os.path.isdir(operator_utils.RAY_CONFIG_DIR): + os.mkdir(operator_utils.RAY_CONFIG_DIR) + # Control loop + cluster_cr_stream = operator_utils.cluster_cr_stream() + try: + for event in cluster_cr_stream: + cluster_cr = event["object"] + event_type = event["type"] + cluster_config = operator_utils.cr_to_config(cluster_cr) + cluster_action(cluster_config, event_type) + except ApiException as e: + if e.status == 404: + raise Exception( + "Caught a 404 error. Has the RayCluster CRD been created?") + else: + raise + + +if __name__ == "__main__": + main() diff --git a/python/ray/operator/operator_utils.py b/python/ray/operator/operator_utils.py new file mode 100644 index 000000000..b6dfd3a26 --- /dev/null +++ b/python/ray/operator/operator_utils.py @@ -0,0 +1,114 @@ +import copy +import logging +import os +from typing import Any, Dict, Iterator, List + +from kubernetes.watch import Watch + +from ray.autoscaler._private.kubernetes import custom_objects_api + +RAY_NAMESPACE = os.environ.get("RAY_OPERATOR_POD_NAMESPACE") + +RAY_CONFIG_DIR = os.path.expanduser("~/ray_cluster_configs") +CONFIG_SUFFIX = "_config.yaml" + +CONFIG_FIELDS = { + "maxWorkers": "max_workers", + "upscalingSpeed": "upscaling_speed", + "idleTimeoutMinutes": "idle_timeout_minutes", + "headPodType": "head_node_type", + "workerDefaultPodType": "worker_default_node_type", + "workerStartRayCommands": "worker_start_ray_commands", + "headStartRayCommands": "head_start_ray_commands", + "podTypes": "available_node_types" +} + +NODE_TYPE_FIELDS = { + "minWorkers": "min_workers", + "maxWorkers": "max_workers", + "podConfig": "node_config", + "rayResources": "resources", + "setupCommands": "worker_setup_commands" +} + +PROVIDER_CONFIG = { + "type": "kubernetes", + "use_internal_ips": True, + "namespace": RAY_NAMESPACE +} + +root_logger = logging.getLogger("ray") +root_logger.setLevel(logging.getLevelName("DEBUG")) +""" +ownerReferences: + - apiVersion: apps/v1 + controller: true + blockOwnerDeletion: true + kind: ReplicaSet + name: my-repset + uid: d9607e19-f88f-11e6-a518-42010a800195 +""" + + +def config_path(cluster_name: str) -> str: + file_name = cluster_name + CONFIG_SUFFIX + return os.path.join(RAY_CONFIG_DIR, file_name) + + +def cluster_cr_stream() -> Iterator: + w = Watch() + return w.stream( + custom_objects_api().list_namespaced_custom_object, + namespace=RAY_NAMESPACE, + group="cluster.ray.io", + version="v1", + plural="rayclusters") + + +def cr_to_config(cluster_resource: Dict[str, Any]) -> Dict[str, Any]: + """Convert RayCluster custom resource to a ray cluster config for use by the + autoscaler.""" + cr_spec = cluster_resource["spec"] + cr_meta = cluster_resource["metadata"] + config = translate(cr_spec, dictionary=CONFIG_FIELDS) + pod_types = cr_spec["podTypes"] + config["available_node_types"] = get_node_types( + pod_types, cluster_name=cr_meta["name"], cluster_uid=cr_meta["uid"]) + config["cluster_name"] = cr_meta["name"] + config["provider"] = PROVIDER_CONFIG + return config + + +def get_node_types(pod_types: List[Dict[str, Any]], cluster_name: str, + cluster_uid: str) -> Dict[str, Any]: + cluster_owner_reference = get_cluster_owner_reference( + cluster_name, cluster_uid) + node_types = {} + for pod_type in pod_types: + name = pod_type["name"] + pod_type_copy = copy.deepcopy(pod_type) + pod_type_copy.pop("name") + node_types[name] = translate( + pod_type_copy, dictionary=NODE_TYPE_FIELDS) + # Deleting a RayCluster CR will also delete the associated pods. + node_types[name]["node_config"]["metadata"].update({ + "ownerReferences": [cluster_owner_reference] + }) + return node_types + + +def get_cluster_owner_reference(cluster_name: str, + cluster_uid: str) -> Dict[str, Any]: + return { + "apiVersion": "apps/v1", + "controller": True, + "blockOwnerDeletion": True, + "kind": "RayCluster", + "name": cluster_name, + "uid": cluster_uid + } + + +def translate(configuration: Dict[str, Any], + dictionary: Dict[str, str]) -> Dict[str, Any]: + return {dictionary[field]: configuration[field] for field in configuration} diff --git a/python/ray/parameter.py b/python/ray/parameter.py index 6c9114603..f6bbc243c 100644 --- a/python/ray/parameter.py +++ b/python/ray/parameter.py @@ -89,7 +89,6 @@ class RayParams: contents to Redis. autoscaling_config: path to autoscaling config file. java_worker_options (list): The command options for Java worker. - load_code_from_local: Whether load code from local file or from GCS. metrics_agent_port(int): The port to bind metrics agent. metrics_export_port(int): The port at which metrics are exposed through a Prometheus endpoint. @@ -142,14 +141,12 @@ class RayParams: include_log_monitor=None, autoscaling_config=None, java_worker_options=None, - load_code_from_local=False, start_initial_python_workers_for_first_job=False, _system_config=None, enable_object_reconstruction=False, metrics_agent_port=None, metrics_export_port=None, - lru_evict=False, - code_search_path=None): + lru_evict=False): self.object_ref_seed = object_ref_seed self.redis_address = redis_address self.num_cpus = num_cpus @@ -186,7 +183,6 @@ class RayParams: self.include_log_monitor = include_log_monitor self.autoscaling_config = autoscaling_config self.java_worker_options = java_worker_options - self.load_code_from_local = load_code_from_local self.metrics_agent_port = metrics_agent_port self.metrics_export_port = metrics_export_port self.start_initial_python_workers_for_first_job = ( @@ -195,9 +191,6 @@ class RayParams: self._lru_evict = lru_evict self._enable_object_reconstruction = enable_object_reconstruction self._check_usage() - self.code_search_path = code_search_path - if code_search_path is None: - self.code_search_path = [] # Set the internal config options for LRU eviction. if lru_evict: diff --git a/python/ray/ray_constants.py b/python/ray/ray_constants.py index 12e429ff5..be717ca3c 100644 --- a/python/ray/ray_constants.py +++ b/python/ray/ray_constants.py @@ -197,7 +197,8 @@ LOG_MONITOR_MAX_OPEN_FILES = 200 # The object metadata field uses the following format: It is a comma # separated list of fields. The first field is mandatory and is the # type of the object (see types below) or an integer, which is interpreted -# as an error value. +# as an error value. The second part is optional and if present has the +# form DEBUG:, it is used for implementing the debugger. # A constant used as object metadata to indicate the object is cross language. OBJECT_METADATA_TYPE_CROSS_LANGUAGE = b"XLANG" @@ -213,6 +214,9 @@ OBJECT_METADATA_TYPE_RAW = b"RAW" # of XLANG. OBJECT_METADATA_TYPE_ACTOR_HANDLE = b"ACTOR_HANDLE" +# A constant indicating the debugging part of the metadata (see above). +OBJECT_METADATA_DEBUG_PREFIX = b"DEBUG:" + AUTOSCALER_RESOURCE_REQUEST_CHANNEL = b"autoscaler_resource_request" # The default password to prevent redis port scanning attack. diff --git a/python/ray/ray_logging.py b/python/ray/ray_logging.py index e5f84eb2a..0668f397f 100644 --- a/python/ray/ray_logging.py +++ b/python/ray/ray_logging.py @@ -153,6 +153,46 @@ class StandardFdRedirectionRotatingFileHandler(RotatingFileHandler): os.dup2(self.stream.fileno(), self.get_original_stream().fileno()) +def get_worker_log_file_name(worker_type): + job_id = os.environ.get("RAY_JOB_ID") + if worker_type == "WORKER": + assert job_id is not None, ( + "RAY_JOB_ID should be set as an env " + "variable within default_worker.py. If you see this error, " + "please report it to Ray's Github issue.") + worker_name = "worker" + else: + job_id = ray.JobID.nil() + worker_name = "io_worker" + + # Make sure these values are set already. + assert ray.worker._global_node is not None + assert ray.worker.global_worker is not None + filename = (f"{worker_name}-" + f"{binary_to_hex(ray.worker.global_worker.worker_id)}-" + f"{job_id}-{os.getpid()}") + return filename + + +def configure_log_file(out_file, err_file): + stdout_fileno = sys.stdout.fileno() + stderr_fileno = sys.stderr.fileno() + # C++ logging requires redirecting the stdout file descriptor. Note that + # dup2 will automatically close the old file descriptor before overriding + # it. + os.dup2(out_file.fileno(), stdout_fileno) + os.dup2(err_file.fileno(), stderr_fileno) + # We also manually set sys.stdout and sys.stderr because that seems to + # have an effect on the output buffering. Without doing this, stdout + # and stderr are heavily buffered resulting in seemingly lost logging + # statements. We never want to close the stdout file descriptor, dup2 will + # close it when necessary and we don't want python's GC to close it. + sys.stdout = ray.utils.open_log( + stdout_fileno, unbuffered=True, closefd=False) + sys.stderr = ray.utils.open_log( + stderr_fileno, unbuffered=True, closefd=False) + + def setup_and_get_worker_interceptor_logger(args, max_bytes=0, backup_count=0, diff --git a/python/ray/remote_function.py b/python/ray/remote_function.py index 68a0eef84..e717e2d28 100644 --- a/python/ray/remote_function.py +++ b/python/ray/remote_function.py @@ -258,8 +258,12 @@ class RemoteFunction: placement_group.id, placement_group_bundle_index, placement_group_capture_child_tasks, + worker.debugger_breakpoint, override_environment_variables=override_environment_variables or dict()) + # Reset worker's debug context from the last "remote" command + # (which applies only to this .remote call). + worker.debugger_breakpoint = b"" if len(object_refs) == 1: return object_refs[0] elif len(object_refs) > 1: diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index 820372076..7d72914ee 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -6,6 +6,7 @@ import logging import os import subprocess import sys +from telnetlib import Telnet import time import urllib import urllib.parse @@ -150,6 +151,35 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port): from None +def continue_debug_session(): + """Continue active debugging session. + + This function will connect 'ray debug' to the right debugger + when a user is stepping between Ray tasks. + """ + active_sessions = ray.experimental.internal_kv._internal_kv_list( + "RAY_PDB_") + + for active_session in active_sessions: + if active_session.startswith(b"RAY_PDB_CONTINUE"): + print("Continuing pdb session in different process...") + key = b"RAY_PDB_" + active_session[len("RAY_PDB_CONTINUE_"):] + while True: + data = ray.experimental.internal_kv._internal_kv_get(key) + if data: + session = json.loads(data) + if "exit_debugger" in session: + ray.experimental.internal_kv._internal_kv_del(key) + return + host, port = session["pdb_address"].split(":") + with Telnet(host, int(port)) as tn: + tn.interact() + ray.experimental.internal_kv._internal_kv_del(key) + continue_debug_session() + return + time.sleep(1.0) + + @cli.command() @click.option( "--address", @@ -158,12 +188,13 @@ def dashboard(cluster_config_file, cluster_name, port, remote_port): help="Override the address to connect to.") def debug(address): """Show all active breakpoints and exceptions in the Ray debugger.""" - from telnetlib import Telnet if not address: address = services.get_ray_address_to_use_or_die() logger.info(f"Connecting to Ray instance at {address}.") - ray.init(address=address) + ray.init(address=address, log_to_driver=False) while True: + continue_debug_session() + active_sessions = ray.experimental.internal_kv._internal_kv_list( "RAY_PDB_") print("Active breakpoints:") @@ -358,25 +389,12 @@ def debug(address): default=None, type=str, help="Overwrite the options to start Java workers.") -@click.option( - "--code-search-path", - default=None, - hidden=True, - type=str, - help="A list of directories or jar files separated by colon that specify " - "the search path for user code. This will be used as `CLASSPATH` in " - "Java and `PYTHONPATH` in Python.") @click.option( "--system-config", default=None, hidden=True, type=json.loads, help="Override system configuration defaults.") -@click.option( - "--load-code-from-local", - is_flag=True, - default=False, - help="Specify whether load code from local file or GCS serialization.") @click.option( "--lru-evict", is_flag=True, @@ -405,8 +423,7 @@ def start(node_ip_address, address, port, redis_password, redis_shard_ports, head, include_dashboard, dashboard_host, dashboard_port, block, plasma_directory, autoscaling_config, no_redirect_worker_output, no_redirect_output, plasma_store_socket_name, raylet_socket_name, - temp_dir, java_worker_options, load_code_from_local, - code_search_path, system_config, lru_evict, + temp_dir, java_worker_options, system_config, lru_evict, enable_object_reconstruction, metrics_export_port, log_style, log_color, verbose): """Start Ray processes manually on the local machine.""" @@ -465,8 +482,6 @@ def start(node_ip_address, address, port, redis_password, redis_shard_ports, dashboard_host=dashboard_host, dashboard_port=dashboard_port, java_worker_options=java_worker_options, - load_code_from_local=load_code_from_local, - code_search_path=code_search_path, _system_config=system_config, lru_evict=lru_evict, enable_object_reconstruction=enable_object_reconstruction, @@ -537,6 +552,8 @@ def start(node_ip_address, address, port, redis_password, redis_shard_ports, with cli_logger.group("Next steps"): cli_logger.print( "To connect to this Ray runtime from another node, run") + # NOTE(kfstorm): Java driver rely on this line to get the address + # of the cluster. Please be careful when updating this line. cli_logger.print( cf.bold(" ray start --address='{}'{}"), redis_address, f" --redis-password='{redis_password}'" @@ -632,7 +649,7 @@ def start(node_ip_address, address, port, redis_password, redis_shard_ports, cli_logger.print( "This command will now block until terminated by a signal.") cli_logger.print( - "Runing subprocesses are monitored and a message will be " + "Running subprocesses are monitored and a message will be " "printed if any of them terminate unexpectedly.") while True: @@ -1273,7 +1290,7 @@ def stack(): COMMAND = """ pyspy=`which py-spy` if [ ! -e "$pyspy" ]; then - echo "ERROR: Please 'pip install py-spy' (or ray[debug]) first" + echo "ERROR: Please 'pip install py-spy' first" exit 1 fi # Set IFS to iterate over lines instead of over words. diff --git a/python/ray/serve/BUILD b/python/ray/serve/BUILD index d1791b613..be8707d86 100644 --- a/python/ray/serve/BUILD +++ b/python/ray/serve/BUILD @@ -17,6 +17,14 @@ py_test( deps = [":serve_lib"], ) +py_test( + name = "test_controller", + size = "small", + srcs = serve_tests_srcs, + tags = ["exclusive"], + deps = [":serve_lib"], +) + py_test( name = "test_backend_worker", size = "small", @@ -35,14 +43,13 @@ py_test( ) -# TODO(simon): Test skipped until #11683 fixed. -# py_test( -# name = "test_failure", -# size = "medium", -# srcs = serve_tests_srcs, -# tags = ["exclusive"], -# deps = [":serve_lib"], -# ) +py_test( + name = "test_failure", + size = "medium", + srcs = serve_tests_srcs, + tags = ["exclusive"], + deps = [":serve_lib"], +) py_test( diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index c5f887c88..ec1593654 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -1,6 +1,9 @@ +import asyncio import atexit +import time from functools import wraps import os +from uuid import UUID import ray from ray.serve.constants import (DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT, @@ -42,6 +45,8 @@ class Client: self._controller_name = controller_name self._detached = detached self._shutdown = False + self._http_host, self._http_port = ray.get( + controller.get_http_config.remote()) # NOTE(simon): Used to cache client.get_handle(endpoint) call. It will # mostly grow in size, it will only shrink when user calls the @@ -62,9 +67,9 @@ class Client: def __del__(self): if not self._detached: - logger.info("Shutting down Ray Serve because client went out of " - "scope. To prevent this, either keep a reference to " - "the client object or use serve.start(detached=True).") + logger.debug("Shutting down Ray Serve because client went out of " + "scope. To prevent this, either keep a reference to " + "the client or use serve.start(detached=True).") self.shutdown() def __reduce__(self): @@ -78,11 +83,34 @@ class Client: Shuts down all processes and deletes all state associated with the instance. """ - if not self._shutdown: + if (not self._shutdown) and ray.is_initialized(): ray.get(self._controller.shutdown.remote()) ray.kill(self._controller, no_restart=True) + + # Wait for the named actor entry gets removed as well. + started = time.time() + while True: + try: + ray.get_actor(self._controller_name) + if time.time() - started > 5: + logger.warning( + "Waited 5s for Serve to shutdown gracefully but " + "the controller is still not cleaned up. " + "You can ignore this warning if you are shutting " + "down the Ray cluster.") + break + except ValueError: # actor name is removed + break + self._shutdown = True + @_ensure_connected + def _get_result(self, result_object_id: ray.ObjectRef) -> bool: + result_id: UUID = ray.get(result_object_id) + result = ray.get(self._controller.wait_for_event.remote(result_id)) + logger.debug(f"Getting result_id ({result_id}) with result: {result}") + return result + @_ensure_connected def create_endpoint(self, endpoint_name: str, @@ -137,10 +165,33 @@ class Client: "an element of type {}".format(type(method))) upper_methods.append(method.upper()) - ray.get( + self._get_result( self._controller.create_endpoint.remote( endpoint_name, {backend: 1.0}, route, upper_methods)) + # Block until the route table has been propagated to all HTTP proxies. + if route is not None: + + def check_ready(http_response): + return route in http_response.json() + + futures = [] + for node_id in ray.state.node_ids(): + future = block_until_http_ready.options( + num_cpus=0, resources={ + node_id: 0.01 + }).remote( + "http://{}:{}/-/routes".format(self._http_host, + self._http_port), + check_ready=check_ready, + timeout=HTTP_PROXY_TIMEOUT) + futures.append(future) + try: + ray.get(futures) + except ray.exceptions.RayTaskError: + raise TimeoutError("Route not available at HTTP proxies " + "after {HTTP_PROXY_TIMEOUT}s.") + @_ensure_connected def delete_endpoint(self, endpoint: str) -> None: """Delete the given endpoint. @@ -149,7 +200,7 @@ class Client: """ if endpoint in self._handle_cache: del self._handle_cache[endpoint] - ray.get(self._controller.delete_endpoint.remote(endpoint)) + self._get_result(self._controller.delete_endpoint.remote(endpoint)) @_ensure_connected def list_endpoints(self) -> Dict[str, Dict[str, Any]]: @@ -193,7 +244,7 @@ class Client: "config_options must be a BackendConfig or dictionary.") if isinstance(config_options, dict): config_options = BackendConfig.parse_obj(config_options) - ray.get( + self._get_result( self._controller.update_backend_config.remote( backend_tag, config_options)) @@ -222,7 +273,8 @@ class Client: Args: backend_tag (str): a unique tag assign to identify this backend. func_or_class (callable, class): a function or a class implementing - __call__. + __call__, returning a JSON-serializable object or a + Starlette Response object. actor_init_args (optional): the arguments to pass to the class. initialization method. ray_actor_options (optional): options to be passed into the @@ -290,7 +342,7 @@ class Client: raise TypeError("config must be a BackendConfig or a dictionary.") backend_config._validate_complete() - ray.get( + self._get_result( self._controller.create_backend.remote(backend_tag, backend_config, replica_config)) @@ -308,7 +360,7 @@ class Client: The backend must not currently be used by any endpoints. """ - ray.get(self._controller.delete_backend.remote(backend_tag)) + self._get_result(self._controller.delete_backend.remote(backend_tag)) @_ensure_connected def set_traffic(self, endpoint_name: str, @@ -327,7 +379,7 @@ class Client: traffic_policy_dictionary (dict): a dictionary maps backend names to their traffic weights. The weights must sum to 1. """ - ray.get( + self._get_result( self._controller.set_traffic.remote(endpoint_name, traffic_policy_dictionary)) @@ -353,20 +405,24 @@ class Client: (float, int)) or not 0 <= proportion <= 1: raise TypeError("proportion must be a float from 0 to 1.") - ray.get( + self._get_result( self._controller.shadow_traffic.remote(endpoint_name, backend_tag, proportion)) @_ensure_connected def get_handle(self, endpoint_name: str, - missing_ok: Optional[bool] = False) -> RayServeHandle: + missing_ok: Optional[bool] = False, + sync: bool = True) -> RayServeHandle: """Retrieve RayServeHandle for service endpoint to invoke it from Python. Args: endpoint_name (str): A registered service endpoint. missing_ok (bool): If true, then Serve won't check the endpoint is registered. False by default. + sync (bool): If true, then Serve will return a ServeHandle that + works everywhere. Otherwise, Serve will return a ServeHandle + that's only usable in asyncio loop. Returns: RayServeHandle @@ -375,8 +431,14 @@ class Client: self._controller.get_all_endpoints.remote()): raise KeyError(f"Endpoint '{endpoint_name}' does not exist.") + if asyncio.get_event_loop().is_running() and sync: + logger.warning( + "You are retrieving a ServeHandle inside an asyncio loop. " + "Try getting client.get_handle(.., sync=False) to get better " + "performance.") + if endpoint_name not in self._handle_cache: - handle = RayServeHandle(self._controller, endpoint_name, sync=True) + handle = RayServeHandle(self._controller, endpoint_name, sync=sync) self._handle_cache[endpoint_name] = handle return self._handle_cache[endpoint_name] @@ -445,7 +507,11 @@ def start(detached: bool = False, "http://{}:{}/-/routes".format(http_host, http_port), timeout=HTTP_PROXY_TIMEOUT) futures.append(future) - ray.get(futures) + try: + ray.get(futures) + except ray.exceptions.RayTaskError: + raise TimeoutError( + "HTTP proxies not available after {HTTP_PROXY_TIMEOUT}s.") return Client(controller, controller_name, detached=detached) diff --git a/python/ray/serve/backend_worker.py b/python/ray/serve/backend_worker.py index 46ee66659..73088b558 100644 --- a/python/ray/serve/backend_worker.py +++ b/python/ray/serve/backend_worker.py @@ -15,10 +15,13 @@ from ray.serve.utils import (parse_request_item, _get_logger, chain_future, from ray.serve.exceptions import RayServeException from ray.util import metrics from ray.serve.config import BackendConfig -from ray.serve.long_poll import LongPollerAsyncClient +from ray.serve.long_poll import LongPollAsyncClient from ray.serve.router import Query -from ray.serve.constants import (DEFAULT_LATENCY_BUCKET_MS, - BACKEND_RECONFIGURE_METHOD) +from ray.serve.constants import ( + BACKEND_RECONFIGURE_METHOD, + DEFAULT_LATENCY_BUCKET_MS, + LongPollKey, +) from ray.exceptions import RayTaskError logger = _get_logger() @@ -168,8 +171,8 @@ class RayServeReplica: tag_keys=("backend", )) self.request_counter.set_default_tags({"backend": self.backend_tag}) - self.long_poll_client = LongPollerAsyncClient(controller_handle, { - "backend_configs": self._update_backend_configs, + self.long_poll_client = LongPollAsyncClient(controller_handle, { + LongPollKey.BACKEND_CONFIGS: self._update_backend_configs, }) self.error_counter = metrics.Count( diff --git a/python/ray/serve/benchmarks/cluster.yaml b/python/ray/serve/benchmarks/cluster.yaml index 831f0f1ff..0a4388533 100644 --- a/python/ray/serve/benchmarks/cluster.yaml +++ b/python/ray/serve/benchmarks/cluster.yaml @@ -1,7 +1,7 @@ cluster_name: default -min_workers: 22 -max_workers: 22 -initial_workers: 22 +min_workers: 5 +max_workers: 5 +initial_workers: 5 autoscaling_mode: default docker: image: 'anyscale/ray-ml:latest' @@ -28,6 +28,7 @@ initialization_commands: [] setup_commands: - apt-get install build-essential libssl-dev git -y - 'rm -r wrk || true && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && cp wrk /usr/local/bin' + - ray install-nightly head_setup_commands: [] worker_setup_commands: [] head_start_ray_commands: diff --git a/python/ray/serve/benchmarks/handle.py b/python/ray/serve/benchmarks/handle.py index b8d243e88..8b168cc3e 100644 --- a/python/ray/serve/benchmarks/handle.py +++ b/python/ray/serve/benchmarks/handle.py @@ -23,64 +23,88 @@ # 2 forwarders and 5 worker replicas: 620 requests/s # 2 forwarders and 10 worker replicas: 609 requests/s +import asyncio +import time + import ray from ray import serve from ray.serve import BackendConfig -from ray.serve.utils import logger -import time -num_queries = 2000 +num_queries = 10000 +max_concurrent_queries = 100000 ray.init(address="auto") -client = serve.start() - -def hello_world(_): +def worker(_): return b"Hello World" class ForwardActor: - def __init__(self): + def __init__(self, sync: bool): client = serve.connect() - self.handle = client.get_handle("hello_world") + self.sync = sync + self.handle = client.get_handle("worker", sync=sync) async def __call__(self, _): - await self.handle.remote() + if self.sync: + await self.handle.remote() + else: + await (await self.handle.remote_async()) -client.create_backend("hello_world", hello_world) -client.create_endpoint("hello_world", backend="hello_world") +async def run_test(num_replicas, num_forwarders, sync): + client = serve.start() + client.create_backend( + "worker", + worker, + config=BackendConfig( + num_replicas=num_replicas, + max_concurrent_queries=max_concurrent_queries, + )) + client.create_endpoint("worker", backend="worker") + endpoint_name = "worker" -client.create_backend("ForwardActor", ForwardActor) -client.create_endpoint("ForwardActor", backend="ForwardActor") + if num_forwarders > 0: + client.create_backend( + "ForwardActor", + ForwardActor, + sync, + config=BackendConfig( + num_replicas=num_forwarders, + max_concurrent_queries=max_concurrent_queries)) + client.create_endpoint("ForwardActor", backend="ForwardActor") + endpoint_name = "ForwardActor" - -def run_test(num_replicas, num_forwarders): - replicas_config = BackendConfig(num_replicas=num_replicas) - client.update_backend_config("hello_world", replicas_config) - - if (num_forwarders == 0): - handle = client.get_handle("hello_world") - else: - forwarders_config = BackendConfig(num_replicas=num_forwarders) - client.update_backend_config("ForwardActor", forwarders_config) - handle = client.get_handle("ForwardActor") + handle = client.get_handle(endpoint_name, sync=sync) # warmup - helpful to wait for gc.collect() and actors to start start = time.time() while time.time() - start < 1: - ray.get(handle.remote()) + if sync: + ray.get(handle.remote()) + else: + ray.get(await handle.remote_async()) # real test start = time.time() - ray.get([handle.remote() for _ in range(num_queries)]) + if sync: + ray.get([handle.remote() for _ in range(num_queries)]) + else: + ray.get([(await handle.remote_async()) for _ in range(num_queries)]) qps = num_queries / (time.time() - start) - logger.info("{} forwarders and {} worker replicas: {} requests/s".format( - num_forwarders, num_replicas, int(qps))) + print( + f"Sync: {sync}, {num_forwarders} forwarders and {num_replicas} worker " + f"replicas: {int(qps)} requests/s") + client.shutdown() -for num_forwarders in [0, 1, 2]: - for num_replicas in [1, 5, 10]: - run_test(num_replicas, num_forwarders) +async def main(): + for sync in [True, False]: + for num_forwarders in [0, 1, 2]: + for num_replicas in [1, 5, 10]: + await run_test(num_replicas, num_forwarders, sync) + + +asyncio.get_event_loop().run_until_complete(main()) diff --git a/python/ray/serve/benchmarks/microbenchmark.py b/python/ray/serve/benchmarks/microbenchmark.py index e4f058e0f..4a34e3418 100644 --- a/python/ray/serve/benchmarks/microbenchmark.py +++ b/python/ray/serve/benchmarks/microbenchmark.py @@ -86,13 +86,14 @@ async def main(): client.create_backend("backend", backend) client.create_endpoint("endpoint", backend="backend", route="/api") for intermediate_handles in [False, True]: - if (intermediate_handles): + if intermediate_handles: client.create_endpoint( "backend", backend="backend", route="/backend") class forwardActor: def __init__(self): + client = serve.connect() self.handle = client.get_handle("backend") def __call__(self, _): diff --git a/python/ray/serve/benchmarks/scalability.py b/python/ray/serve/benchmarks/scalability.py index d11564567..c424ae321 100644 --- a/python/ray/serve/benchmarks/scalability.py +++ b/python/ray/serve/benchmarks/scalability.py @@ -36,73 +36,76 @@ from ray import serve from ray.serve import BackendConfig from ray.serve.utils import logger -from ray.util.placement_group import (placement_group, remove_placement_group) +from ray.util.placement_group import placement_group, remove_placement_group ray.shutdown() ray.init(address="auto") -client = serve.start() -# These numbers need to correspond with the autoscaler config file. -# The number of remote nodes in the autoscaler should upper bound -# these because sometimes nodes fail to update. -num_workers = 20 -expected_num_nodes = num_workers + 1 -cpus_per_node = 4 -num_remote_cpus = expected_num_nodes * cpus_per_node +# We ask for more worker but only need to run on smaller subset. +# This should account for worker nodes failed to launch. +expected_num_nodes = 6 +num_replicas = 11 +# wrk HTTP load testing config +num_connections = 20 +num_threads = 2 +time_to_run = "20s" # Wait until the expected number of nodes have joined the cluster. while True: - num_nodes = len(ray.nodes()) + num_nodes = len(list(filter(lambda node: node["Alive"], ray.nodes()))) logger.info("Waiting for nodes {}/{}".format(num_nodes, expected_num_nodes)) if num_nodes >= expected_num_nodes: break time.sleep(5) + logger.info("Nodes have all joined. There are %s resources.", ray.cluster_resources()) +client = serve.start() + def hey(_): time.sleep(0.01) # Sleep for 10ms return b"hey" -num_connections = int(num_remote_cpus * 0.75) -num_threads = 2 -time_to_run = "10s" - pg = placement_group( [{ "CPU": 1 } for _ in range(expected_num_nodes)], strategy="STRICT_SPREAD") ray.get(pg.ready()) -# The number of replicas is the number of cores remaining after accounting -# for the one HTTP proxy actor on each node, the "hey" requester task on each -# node, and the serve controller. -# num_replicas = expected_num_nodes * (cpus_per_node - 2) - 1 -num_replicas = ray.available_resources()["CPU"] logger.info("Starting %i replicas", num_replicas) client.create_backend( "hey", hey, config=BackendConfig(num_replicas=num_replicas)) client.create_endpoint("hey", backend="hey", route="/hey") -@ray.remote +@ray.remote(num_cpus=0) def run_wrk(): - logger.info("Warming up for ~3 seconds") - for _ in range(5): - resp = requests.get("http://127.0.0.1:8000/hey").text - logger.info("Received response \'" + resp + "\'") - time.sleep(0.5) + logger.info("Warming up") + for _ in range(10): + try: + resp = requests.get("http://127.0.0.1:8000/hey").text + logger.info("Received response '" + resp + "'") + time.sleep(0.5) + except Exception as e: + logger.info(f"Got exception {e}") result = subprocess.run( [ - "wrk", "-c", - str(num_connections), "-t", - str(num_threads), "-d", time_to_run, "http://127.0.0.1:8000/hey" + "wrk", + "-c", + str(num_connections), + "-t", + str(num_threads), + "-d", + time_to_run, + "http://127.0.0.1:8000/hey", ], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + ) return result.stdout.decode() diff --git a/python/ray/serve/benchmarks/single.yaml b/python/ray/serve/benchmarks/single.yaml index b93577fca..4500d65ab 100644 --- a/python/ray/serve/benchmarks/single.yaml +++ b/python/ray/serve/benchmarks/single.yaml @@ -23,6 +23,7 @@ initialization_commands: [] setup_commands: - apt-get install build-essential libssl-dev git -y - 'rm -r wrk || true && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && cp wrk /usr/local/bin' + - ray install-nightly head_setup_commands: [] worker_setup_commands: [] head_start_ray_commands: diff --git a/python/ray/serve/constants.py b/python/ray/serve/constants.py index 52c76972f..db2b2008a 100644 --- a/python/ray/serve/constants.py +++ b/python/ray/serve/constants.py @@ -1,3 +1,5 @@ +from enum import auto, Enum + #: Actor name used to register controller SERVE_CONTROLLER_NAME = "SERVE_CONTROLLER_ACTOR" @@ -37,3 +39,13 @@ DEFAULT_LATENCY_BUCKET_MS = [ #: Name of backend reconfiguration method implemented by user. BACKEND_RECONFIGURE_METHOD = "reconfigure" + + +class LongPollKey(Enum): + def __repr__(self): + return f"{self.__class__.__name__}.{self.name}" + + REPLICA_HANDLES = auto() + TRAFFIC_POLICIES = auto() + BACKEND_CONFIGS = auto() + ROUTE_TABLE = auto() diff --git a/python/ray/serve/controller.py b/python/ray/serve/controller.py index 3176a6321..5237d453c 100644 --- a/python/ray/serve/controller.py +++ b/python/ray/serve/controller.py @@ -6,20 +6,22 @@ import random import time from dataclasses import dataclass, field from typing import Dict, Any, List, Optional, Tuple +from uuid import uuid4, UUID from pydantic import BaseModel import ray import ray.cloudpickle as pickle from ray.serve.autoscaling_policy import BasicAutoscalingPolicy from ray.serve.backend_worker import create_backend_replica -from ray.serve.constants import ASYNC_CONCURRENCY, SERVE_PROXY_NAME +from ray.serve.constants import (ASYNC_CONCURRENCY, SERVE_PROXY_NAME, + LongPollKey) from ray.serve.http_proxy import HTTPProxyActor from ray.serve.kv_store import RayInternalKVStore from ray.serve.exceptions import RayServeException from ray.serve.utils import (format_actor_name, get_random_letters, logger, try_schedule_resources_on_nodes, get_all_node_ids) from ray.serve.config import BackendConfig, ReplicaConfig -from ray.serve.long_poll import LongPollerHost +from ray.serve.long_poll import LongPollHost from ray.actor import ActorHandle import numpy as np @@ -144,7 +146,7 @@ class ActorStateReconciler: controller_name: str = field(init=True) detached: bool = field(init=True) - routers_cache: Dict[NodeId, ActorHandle] = field(default_factory=dict) + http_proxy_cache: Dict[NodeId, ActorHandle] = field(default_factory=dict) backend_replicas: Dict[BackendTag, Dict[ReplicaTag, ActorHandle]] = field( default_factory=lambda: defaultdict(dict)) backend_replicas_to_start: Dict[BackendTag, List[ReplicaTag]] = field( @@ -156,8 +158,8 @@ class ActorStateReconciler: # TODO(edoakes): consider removing this and just using the names. - def router_handles(self) -> List[ActorHandle]: - return list(self.routers_cache.values()) + def http_proxy_handles(self) -> List[ActorHandle]: + return list(self.http_proxy_cache.values()) def get_replica_handles(self) -> List[ActorHandle]: return list( @@ -302,7 +304,7 @@ class ActorStateReconciler: async def _stop_pending_backend_replicas(self) -> None: """Stops the pending backend replicas in self.backend_replicas_to_stop. - Removes backend_replicas from the router, kills them, and clears + Removes backend_replicas from the http_proxy, kills them, and clears self.backend_replicas_to_stop. """ for backend_tag, replicas_list in self.backend_replicas_to_stop.items( @@ -326,26 +328,26 @@ class ActorStateReconciler: self.backend_replicas_to_stop.clear() - def _start_routers_if_needed(self, http_host: str, http_port: str, - http_middlewares: List[Any]) -> None: - """Start a router on every node if it doesn't already exist.""" + def _start_http_proxies_if_needed(self, http_host: str, http_port: str, + http_middlewares: List[Any]) -> None: + """Start an HTTP proxy on every node if it doesn't already exist.""" if http_host is None: return for node_id, node_resource in get_all_node_ids(): - if node_id in self.routers_cache: + if node_id in self.http_proxy_cache: continue - router_name = format_actor_name(SERVE_PROXY_NAME, - self.controller_name, node_id) + name = format_actor_name(SERVE_PROXY_NAME, self.controller_name, + node_id) try: - router = ray.get_actor(router_name) + proxy = ray.get_actor(name) except ValueError: - logger.info("Starting router with name '{}' on node '{}' " + logger.info("Starting HTTP proxy with name '{}' on node '{}' " "listening on '{}:{}'".format( - router_name, node_id, http_host, http_port)) - router = HTTPProxyActor.options( - name=router_name, + name, node_id, http_host, http_port)) + proxy = HTTPProxyActor.options( + name=name, lifetime="detached" if self.detached else None, max_concurrency=ASYNC_CONCURRENCY, max_restarts=-1, @@ -359,10 +361,10 @@ class ActorStateReconciler: controller_name=self.controller_name, http_middlewares=http_middlewares) - self.routers_cache[node_id] = router + self.http_proxy_cache[node_id] = proxy - def _stop_routers_if_needed(self) -> bool: - """Removes router actors from any nodes that no longer exist. + def _stop_http_proxies_if_needed(self) -> bool: + """Removes HTTP proxy actors from any nodes that no longer exist. Returns whether or not any actors were removed (a checkpoint should be taken). @@ -370,25 +372,25 @@ class ActorStateReconciler: actor_stopped = False all_node_ids = {node_id for node_id, _ in get_all_node_ids()} to_stop = [] - for node_id in self.routers_cache: + for node_id in self.http_proxy_cache: if node_id not in all_node_ids: - logger.info( - "Removing router on removed node '{}'.".format(node_id)) + logger.info("Removing HTTP proxy on removed node '{}'.".format( + node_id)) to_stop.append(node_id) for node_id in to_stop: - router_handle = self.routers_cache.pop(node_id) - ray.kill(router_handle, no_restart=True) + proxy = self.http_proxy_cache.pop(node_id) + ray.kill(proxy, no_restart=True) actor_stopped = True return actor_stopped def _recover_actor_handles(self) -> None: # Refresh the RouterCache - for node_id in self.routers_cache.keys(): - router_name = format_actor_name(SERVE_PROXY_NAME, - self.controller_name, node_id) - self.routers_cache[node_id] = ray.get_actor(router_name) + for node_id in self.http_proxy_cache.keys(): + name = format_actor_name(SERVE_PROXY_NAME, self.controller_name, + node_id) + self.http_proxy_cache[node_id] = ray.get_actor(name) # Fetch actor handles for all of the backend replicas in the system. # All of these backend_replicas are guaranteed to already exist because @@ -420,12 +422,19 @@ class ActorStateReconciler: return autoscaling_policies +@dataclass +class FutureResult: + # Goal requested when this future was created + requested_goal: Dict[str, Any] + + @dataclass class Checkpoint: goal_state: SystemState current_state: SystemState reconciler: ActorStateReconciler # TODO(ilr) Rename reconciler to PendingState + inflight_reqs: Dict[uuid4, FutureResult] @ray.remote @@ -474,7 +483,7 @@ class ServeController: # backend -> AutoscalingPolicy self.autoscaling_policies = dict() - # Dictionary of backend_tag -> router_name -> most recent queue length. + # Dictionary of backend_tag -> proxy_name -> most recent queue length. self.backend_stats = defaultdict(lambda: defaultdict(dict)) # Used to ensure that only a single state-changing operation happens @@ -487,56 +496,87 @@ class ServeController: # If starting the actor for the first time, starts up the other system # components. If recovering, fetches their actor handles. - self.actor_reconciler._start_routers_if_needed( + self.actor_reconciler._start_http_proxies_if_needed( self.http_host, self.http_port, self.http_middlewares) - # NOTE(edoakes): unfortunately, we can't completely recover from a - # checkpoint in the constructor because we block while waiting for - # other actors to start up, and those actors fetch soft state from - # this actor. Because no other tasks will start executing until after - # the constructor finishes, if we were to run this logic in the - # constructor it could lead to deadlock between this actor and a child. - # However we do need to guarantee that we have fully recovered from a - # checkpoint before any other state-changing calls run. We address this - # by acquiring the write_lock and then posting the task to recover from - # a checkpoint to the event loop. Other state-changing calls acquire - # this lock and will be blocked until recovering from the checkpoint - # finishes. + # Map of awaiting results + # TODO(ilr): Checkpoint this once this becomes asynchronous + self.inflight_results: Dict[UUID, asyncio.Event] = dict() + self._serializable_inflight_results: Dict[UUID, FutureResult] = dict() + checkpoint = self.kv_store.get(CHECKPOINT_KEY) if checkpoint is None: logger.debug("No checkpoint found") else: - await self.write_lock.acquire() - asyncio.get_event_loop().create_task( - self._recover_from_checkpoint(checkpoint)) + await self._recover_from_checkpoint(checkpoint) # NOTE(simon): Currently we do all-to-all broadcast. This means # any listeners will receive notification for all changes. This # can be problem at scale, e.g. updating a single backend config # will send over the entire configs. In the future, we should # optimize the logic to support subscription by key. - self.long_poll_host = LongPollerHost() + self.long_poll_host = LongPollHost() + + # The configs pushed out here get updated by + # self._recover_from_checkpoint in the failure scenario, so that must + # be run before we notify the changes. self.notify_backend_configs_changed() self.notify_replica_handles_changed() self.notify_traffic_policies_changed() + self.notify_route_table_changed() asyncio.get_event_loop().create_task(self.run_control_loop()) + async def wait_for_event(self, uuid: UUID) -> bool: + if uuid not in self.inflight_results: + return True + event = self.inflight_results[uuid] + await event.wait() + self.inflight_results.pop(uuid) + self._serializable_inflight_results.pop(uuid) + async with self.write_lock: + self._checkpoint() + + return True + + def _create_event_with_result( + self, + goal_state: Dict[str, any], + recreation_uuid: Optional[UUID] = None) -> UUID: + # NOTE(ilr) Must be called before checkpointing! + event = asyncio.Event() + event.result = FutureResult(goal_state) + event.set() + uuid_val = recreation_uuid or uuid4() + self.inflight_results[uuid_val] = event + self._serializable_inflight_results[uuid_val] = event.result + return uuid_val + + async def _num_inflight_results(self) -> int: + return len(self.inflight_results) + def notify_replica_handles_changed(self): self.long_poll_host.notify_changed( - "worker_handles", { + LongPollKey.REPLICA_HANDLES, { backend_tag: list(replica_dict.values()) for backend_tag, replica_dict in self.actor_reconciler.backend_replicas.items() }) def notify_traffic_policies_changed(self): - self.long_poll_host.notify_changed("traffic_policies", - self.current_state.traffic_policies) + self.long_poll_host.notify_changed( + LongPollKey.TRAFFIC_POLICIES, + self.current_state.traffic_policies, + ) def notify_backend_configs_changed(self): self.long_poll_host.notify_changed( - "backend_configs", self.current_state.get_backend_configs()) + LongPollKey.BACKEND_CONFIGS, + self.current_state.get_backend_configs()) + + def notify_route_table_changed(self): + self.long_poll_host.notify_changed(LongPollKey.ROUTE_TABLE, + self.current_state.routes) async def listen_for_change(self, keys_to_snapshot_ids: Dict[str, int]): """Proxy long pull client's listen request. @@ -549,13 +589,9 @@ class ServeController: return await ( self.long_poll_host.listen_for_change(keys_to_snapshot_ids)) - def get_routers(self) -> Dict[str, ActorHandle]: - """Returns a dictionary of node ID to router actor handles.""" - return self.actor_reconciler.routers_cache - - def get_router_config(self) -> Dict[str, Tuple[str, List[str]]]: - """Called by the router on startup to fetch required state.""" - return self.current_state.routes + def get_http_proxies(self) -> Dict[str, ActorHandle]: + """Returns a dictionary of node ID to http_proxy actor handles.""" + return self.actor_reconciler.http_proxy_cache def _checkpoint(self) -> None: """Checkpoint internal state and write it to the KV store.""" @@ -565,7 +601,8 @@ class ServeController: checkpoint = pickle.dumps( Checkpoint(self.goal_state, self.current_state, - self.actor_reconciler)) + self.actor_reconciler, + self._serializable_inflight_results)) self.kv_store.put(CHECKPOINT_KEY, checkpoint) logger.debug("Wrote checkpoint in {:.2f}".format(time.time() - start)) @@ -578,35 +615,51 @@ class ServeController: async def _recover_from_checkpoint(self, checkpoint_bytes: bytes) -> None: """Recover the instance state from the provided checkpoint. + This should be called in the constructor to ensure that the internal + state is updated before any other operations run. After running this, + internal state will be updated and long-poll clients may be notified. + Performs the following operations: 1) Deserializes the internal state from the checkpoint. - 2) Pushes the latest configuration to the routers - in case we crashed before updating them. - 3) Starts/stops any replicas that are pending creation or + 2) Starts/stops any replicas that are pending creation or deletion. - - NOTE: this requires that self.write_lock is already acquired and will - release it before returning. """ - assert self.write_lock.locked() - start = time.time() logger.info("Recovering from checkpoint") restored_checkpoint: Checkpoint = pickle.loads(checkpoint_bytes) - # Restore SystemState self.current_state = restored_checkpoint.current_state - # Restore ActorStateReconciler self.actor_reconciler = restored_checkpoint.reconciler - self.autoscaling_policies = await self.actor_reconciler.\ - _recover_from_checkpoint(self.current_state, self) + self._serializable_inflight_results = restored_checkpoint.inflight_reqs + for uuid, fut_result in self._serializable_inflight_results.items(): + self._create_event_with_result(fut_result.requested_goal, uuid) - logger.info( - "Recovered from checkpoint in {:.3f}s".format(time.time() - start)) + # NOTE(edoakes): unfortunately, we can't completely recover from a + # checkpoint in the constructor because we block while waiting for + # other actors to start up, and those actors fetch soft state from + # this actor. Because no other tasks will start executing until after + # the constructor finishes, if we were to run this logic in the + # constructor it could lead to deadlock between this actor and a child. + # However, we do need to guarantee that we have fully recovered from a + # checkpoint before any other state-changing calls run. We address this + # by acquiring the write_lock and then posting the task to recover from + # a checkpoint to the event loop. Other state-changing calls acquire + # this lock and will be blocked until recovering from the checkpoint + # finishes. This can be removed once we move to the async control loop. - self.write_lock.release() + async def finish_recover_from_checkpoint(): + assert self.write_lock.locked() + self.autoscaling_policies = await self.actor_reconciler.\ + _recover_from_checkpoint(self.current_state, self) + self.write_lock.release() + logger.info( + "Recovered from checkpoint in {:.3f}s".format(time.time() - + start)) + + await self.write_lock.acquire() + asyncio.get_event_loop().create_task(finish_recover_from_checkpoint()) async def do_autoscale(self) -> None: for backend, info in self.current_state.backends.items(): @@ -623,44 +676,30 @@ class ServeController: while True: await self.do_autoscale() async with self.write_lock: - self.actor_reconciler._start_routers_if_needed( + self.actor_reconciler._start_http_proxies_if_needed( self.http_host, self.http_port, self.http_middlewares) checkpoint_required = self.actor_reconciler.\ - _stop_routers_if_needed() + _stop_http_proxies_if_needed() if checkpoint_required: self._checkpoint() await asyncio.sleep(CONTROL_LOOP_PERIOD_S) - def get_backend_configs(self) -> Dict[str, BackendConfig]: - """Fetched by the router on startup.""" - return self.current_state.get_backend_configs() - - def get_traffic_policies(self) -> Dict[str, TrafficPolicy]: - """Fetched by the router on startup.""" - return self.current_state.traffic_policies - - def _list_replicas(self, backend_tag: BackendTag) -> List[ReplicaTag]: - """Used only for testing.""" - return list(self.actor_reconciler.backend_replicas[backend_tag].keys()) - - def get_traffic_policy(self, endpoint: str) -> TrafficPolicy: - """Fetched by serve handles.""" - return self.current_state.traffic_policies[endpoint] - - def get_all_replica_handles(self) -> Dict[str, Dict[str, ActorHandle]]: - """Fetched by the router on startup.""" + def _all_replica_handles( + self) -> Dict[BackendTag, Dict[ReplicaTag, ActorHandle]]: + """Used for testing.""" return self.actor_reconciler.backend_replicas - def get_all_backends(self) -> Dict[str, BackendConfig]: + def get_all_backends(self) -> Dict[BackendTag, BackendConfig]: """Returns a dictionary of backend tag to backend config.""" return self.current_state.get_backend_configs() - def get_all_endpoints(self) -> Dict[str, Dict[str, Any]]: + def get_all_endpoints(self) -> Dict[EndpointTag, Dict[BackendTag, Any]]: + """Returns a dictionary of backend tag to backend config.""" return self.current_state.get_endpoints() async def _set_traffic(self, endpoint_name: str, - traffic_dict: Dict[str, float]) -> None: + traffic_dict: Dict[str, float]) -> UUID: if endpoint_name not in self.current_state.get_endpoints(): raise ValueError("Attempted to assign traffic for an endpoint '{}'" " that is not registered.".format(endpoint_name)) @@ -677,21 +716,25 @@ class ServeController: traffic_policy = TrafficPolicy(traffic_dict) self.current_state.traffic_policies[endpoint_name] = traffic_policy + return_uuid = self._create_event_with_result({ + endpoint_name: traffic_policy + }) # NOTE(edoakes): we must write a checkpoint before pushing the # update to avoid inconsistent state if we crash after pushing the # update. self._checkpoint() - self.notify_traffic_policies_changed() + return return_uuid async def set_traffic(self, endpoint_name: str, - traffic_dict: Dict[str, float]) -> None: + traffic_dict: Dict[str, float]) -> UUID: """Sets the traffic policy for the specified endpoint.""" async with self.write_lock: - await self._set_traffic(endpoint_name, traffic_dict) + return_uuid = await self._set_traffic(endpoint_name, traffic_dict) + return return_uuid async def shadow_traffic(self, endpoint_name: str, backend_tag: BackendTag, - proportion: float) -> None: + proportion: float) -> UUID: """Shadow traffic from the endpoint to the backend.""" async with self.write_lock: if endpoint_name not in self.current_state.get_endpoints(): @@ -707,16 +750,22 @@ class ServeController: self.current_state.traffic_policies[endpoint_name].set_shadow( backend_tag, proportion) + traffic_policy = self.current_state.traffic_policies[endpoint_name] + + return_uuid = self._create_event_with_result({ + endpoint_name: traffic_policy + }) # NOTE(edoakes): we must write a checkpoint before pushing the # update to avoid inconsistent state if we crash after pushing the # update. self._checkpoint() self.notify_traffic_policies_changed() + return return_uuid # TODO(architkulkarni): add Optional for route after cloudpickle upgrade async def create_endpoint(self, endpoint: str, traffic_dict: Dict[str, float], route, - methods) -> None: + methods) -> UUID: """Create a new endpoint with the specified route and methods. If the route is None, this is a "headless" endpoint that will not @@ -755,13 +804,11 @@ class ServeController: self.current_state.routes[route] = (endpoint, methods) # NOTE(edoakes): checkpoint is written in self._set_traffic. - await self._set_traffic(endpoint, traffic_dict) - await asyncio.gather(*[ - router.set_route_table.remote(self.current_state.routes) - for router in self.actor_reconciler.router_handles() - ]) + return_uuid = await self._set_traffic(endpoint, traffic_dict) + self.notify_route_table_changed() + return return_uuid - async def delete_endpoint(self, endpoint: str) -> None: + async def delete_endpoint(self, endpoint: str) -> UUID: """Delete the specified endpoint. Does not modify any corresponding backends. @@ -788,19 +835,20 @@ class ServeController: self.actor_reconciler.endpoints_to_remove.append(endpoint) + return_uuid = self._create_event_with_result({ + route_to_delete: None, + endpoint: None + }) # NOTE(edoakes): we must write a checkpoint before pushing the - # updates to the routers to avoid inconsistent state if we crash + # updates to the proxies to avoid inconsistent state if we crash # after pushing the update. self._checkpoint() - - await asyncio.gather(*[ - router.set_route_table.remote(self.current_state.routes) - for router in self.actor_reconciler.router_handles() - ]) + self.notify_route_table_changed() + return return_uuid async def create_backend(self, backend_tag: BackendTag, backend_config: BackendConfig, - replica_config: ReplicaConfig) -> None: + replica_config: ReplicaConfig) -> UUID: """Register a new backend under the specified tag.""" async with self.write_lock: # Ensures this method is idempotent. @@ -815,12 +863,11 @@ class ServeController: # Save creator that starts replicas, the arguments to be passed in, # and the configuration for the backends. - self.current_state.add_backend( - backend_tag, - BackendInfo( - worker_class=backend_replica, - backend_config=backend_config, - replica_config=replica_config)) + backend_info = BackendInfo( + worker_class=backend_replica, + backend_config=backend_config, + replica_config=replica_config) + self.current_state.add_backend(backend_tag, backend_info) metadata = backend_config.internal_metadata if metadata.autoscaling_config is not None: self.autoscaling_policies[ @@ -835,6 +882,9 @@ class ServeController: del self.current_state.backends[backend_tag] raise e + return_uuid = self._create_event_with_result({ + backend_tag: backend_info + }) # NOTE(edoakes): we must write a checkpoint before starting new # or pushing the updated config to avoid inconsistent state if we # crash while making the change. @@ -844,11 +894,12 @@ class ServeController: self.notify_replica_handles_changed() - # Set the backend config inside the router + # Set the backend config inside routers # (particularly for max_concurrent_queries). self.notify_backend_configs_changed() + return return_uuid - async def delete_backend(self, backend_tag: BackendTag) -> None: + async def delete_backend(self, backend_tag: BackendTag) -> UUID: async with self.write_lock: # This method must be idempotent. We should validate that the # specified backend exists on the client. @@ -876,19 +927,21 @@ class ServeController: if backend_tag in self.autoscaling_policies: del self.autoscaling_policies[backend_tag] - # Add the intention to remove the backend from the router. + # Add the intention to remove the backend from the routers. self.actor_reconciler.backends_to_remove.append(backend_tag) + return_uuid = self._create_event_with_result({backend_tag: None}) # NOTE(edoakes): we must write a checkpoint before removing the - # backend from the router to avoid inconsistent state if we crash + # backend from the routers to avoid inconsistent state if we crash # after pushing the update. self._checkpoint() await self.actor_reconciler._stop_pending_backend_replicas() self.notify_replica_handles_changed() + return return_uuid async def update_backend_config(self, backend_tag: BackendTag, - config_options: BackendConfig) -> None: + config_options: BackendConfig) -> UUID: """Set the config for the specified backend.""" async with self.write_lock: assert (self.current_state.get_backend(backend_tag) @@ -902,18 +955,22 @@ class ServeController: backend_config._validate_complete() self.current_state.get_backend( backend_tag).backend_config = backend_config + backend_info = self.current_state.get_backend(backend_tag) # Scale the replicas with the new configuration. self.actor_reconciler._scale_backend_replicas( self.current_state.backends, backend_tag, backend_config.num_replicas) + return_uuid = self._create_event_with_result({ + backend_tag: backend_info + }) # NOTE(edoakes): we must write a checkpoint before pushing the # update to avoid inconsistent state if we crash after pushing the # update. self._checkpoint() - # Inform the router about change in configuration + # Inform the routers about change in configuration # (particularly for setting max_batch_size). await self.actor_reconciler._start_pending_backend_replicas( @@ -922,6 +979,7 @@ class ServeController: self.notify_replica_handles_changed() self.notify_backend_configs_changed() + return return_uuid def get_backend_config(self, backend_tag: BackendTag) -> BackendConfig: """Get the current config for the specified backend.""" @@ -929,11 +987,15 @@ class ServeController: ), "Backend {} is not registered.".format(backend_tag) return self.current_state.get_backend(backend_tag).backend_config + def get_http_config(self): + """Return the HTTP proxy configuration.""" + return self.http_host, self.http_port + async def shutdown(self) -> None: """Shuts down the serve instance completely.""" async with self.write_lock: - for router in self.actor_reconciler.router_handles(): - ray.kill(router, no_restart=True) + for http_proxy in self.actor_reconciler.http_proxy_handles(): + ray.kill(http_proxy, no_restart=True) for replica in self.actor_reconciler.get_replica_handles(): ray.kill(replica, no_restart=True) self.kv_store.delete(CHECKPOINT_KEY) diff --git a/python/ray/serve/endpoint_policy.py b/python/ray/serve/endpoint_policy.py index 555c4f58a..c757f20c3 100644 --- a/python/ray/serve/endpoint_policy.py +++ b/python/ray/serve/endpoint_policy.py @@ -89,5 +89,6 @@ class RandomEndpointPolicy(EndpointPolicy): query.metadata.shard_key.encode("utf-8")) chosen_backend, shadow_backends = self._select_backends(value) - logger.debug(f"Chosen backend {chosen_backend} for query {query}") + logger.debug(f"Assigning query {query.metadata.request_id} " + f"to backend {chosen_backend}.") return [chosen_backend] + shadow_backends diff --git a/python/ray/serve/handle.py b/python/ray/serve/handle.py index adcd5ac53..9e88959c8 100644 --- a/python/ray/serve/handle.py +++ b/python/ray/serve/handle.py @@ -7,6 +7,7 @@ import ray from ray.serve.context import TaskContext from ray.serve.router import RequestMetadata, Router from ray.serve.utils import get_random_letters +from ray.serve.exceptions import RayServeException global_async_loop = None @@ -109,16 +110,25 @@ class RayServeHandle: ``**kwargs``: All keyword arguments will be available in ``request.args``. """ - assert self.sync, "handle.remote() should be called from sync handle." + if not self.sync: + raise RayServeException( + "You are trying to call handle.remote() with async handle. " + "Please use `await handle.remote_async()` instead.") + coro = self._remote(request_data, kwargs) future: concurrent.futures.Future = asyncio.run_coroutine_threadsafe( coro, self.async_loop) + # Block until the result is ready. return future.result() - async def _remote_async(self, request_data, **kwargs) -> ray.ObjectRef: + async def remote_async(self, + request_data: Optional[Union[Dict, Any]] = None, + **kwargs) -> ray.ObjectRef: """Experimental API for enqueue a request in async context.""" - assert not self.sync, "_remote_async must be called inside async loop." + if not asyncio.get_event_loop().is_running(): + raise RayServeException( + "remote_async must be called from a running event loop.") return await self._remote(request_data, kwargs) def options(self, diff --git a/python/ray/serve/http_proxy.py b/python/ray/serve/http_proxy.py index 9432deefb..3215f3578 100644 --- a/python/ray/serve/http_proxy.py +++ b/python/ray/serve/http_proxy.py @@ -3,46 +3,46 @@ import socket from typing import List import uvicorn +import starlette.responses import ray from ray.exceptions import RayTaskError +from ray.serve.constants import LongPollKey from ray.serve.context import TaskContext from ray.util import metrics from ray.serve.utils import _get_logger, get_random_letters from ray.serve.http_util import Response +from ray.serve.long_poll import LongPollAsyncClient from ray.serve.router import Router, RequestMetadata -# The maximum number of times to retry a request due to actor failure. -# TODO(edoakes): this should probably be configurable. -MAX_ACTOR_DEAD_RETRIES = 10 - logger = _get_logger() class HTTPProxy: - """ - This class should be instantiated and ran by ASGI server. + """This class is meant to be instantiated and run by an ASGI HTTP server. >>> import uvicorn >>> uvicorn.run(HTTPProxy(kv_store_actor_handle, router_handle)) - # blocks forever """ - async def fetch_config_from_controller(self, controller_name): - assert ray.is_initialized() + def __init__(self, controller_name): controller = ray.get_actor(controller_name) - - self.route_table = await controller.get_router_config.remote() + self.route_table = {} # Should be updated via long polling. + self.router = Router(controller) + self.long_poll_client = LongPollAsyncClient(controller, { + LongPollKey.ROUTE_TABLE: self._update_route_table, + }) self.request_counter = metrics.Count( "num_http_requests", description="The number of HTTP requests processed", tag_keys=("route", )) - self.router = Router(controller) + async def setup(self): await self.router.setup_in_async_loop() - def set_route_table(self, route_table): + async def _update_route_table(self, route_table): + logger.debug(f"HTTP Proxy: Get updated route table: {route_table}.") self.route_table = route_table async def receive_http_body(self, scope, receive, send): @@ -74,8 +74,11 @@ class HTTPProxy: status_code=404).send(scope, receive, send) async def __call__(self, scope, receive, send): - # NOTE: This implements ASGI protocol specified in - # https://asgi.readthedocs.io/en/latest/specs/index.html + """Implements the ASGI protocol. + + See details at: + https://asgi.readthedocs.io/en/latest/specs/index.html. + """ error_sender = self._make_error_sender(scope, receive, send) @@ -126,6 +129,18 @@ class HTTPProxy: if isinstance(result, RayTaskError): error_message = "Task Error. Traceback: {}.".format(result) await error_sender(error_message, 500) + elif isinstance(result, starlette.responses.Response): + if isinstance(result, starlette.responses.StreamingResponse): + raise TypeError("Starlette StreamingResponse returned by " + f"backend for endpoint {endpoint_name}. " + "StreamingResponse is unserializable and not " + "supported by Ray Serve. Consider using " + "another Starlette response type such as " + "Response, HTMLResponse, PlainTextResponse, " + "or JSONResponse. If support for " + "StreamingResponse is desired, please let " + "the Ray team know by making a Github issue!") + await result(scope, receive, send) else: await Response(result).send(scope, receive, send) @@ -137,12 +152,13 @@ class HTTPProxyActor: host, port, controller_name, - http_middlewares: List["starlette.middleware.Middleware"] = []): + http_middlewares: List[ + "starlette.middleware.Middleware"] = []): # noqa: F821 self.host = host self.port = port - self.app = HTTPProxy() - await self.app.fetch_config_from_controller(controller_name) + self.app = HTTPProxy(controller_name) + await self.app.setup() self.wrapped_app = self.app for middleware in http_middlewares: @@ -180,12 +196,3 @@ class HTTPProxyActor: # the main thread and uvicorn doesn't expose a way to configure it. server.install_signal_handlers = lambda: None await server.serve(sockets=[sock]) - - async def set_route_table(self, route_table): - self.app.set_route_table(route_table) - - # ------ Proxy router logic ------ # - async def assign_request(self, request_meta, *request_args, - **request_kwargs): - return await (await self.app.router.assign_request( - request_meta, *request_args, **request_kwargs)) diff --git a/python/ray/serve/http_util.py b/python/ray/serve/http_util.py index da41c10eb..1a057a88e 100644 --- a/python/ray/serve/http_util.py +++ b/python/ray/serve/http_util.py @@ -117,7 +117,7 @@ class Response: elif content_type == "json": self.raw_headers.append([b"content-type", b"application/json"]) else: - raise ValueError("Invalid content type {}".foramt(content_type)) + raise ValueError("Invalid content type {}".format(content_type)) async def send(self, scope, receive, send): await send({ diff --git a/python/ray/serve/long_poll.py b/python/ray/serve/long_poll.py index 2d747e94e..01df2cfe0 100644 --- a/python/ray/serve/long_poll.py +++ b/python/ray/serve/long_poll.py @@ -1,4 +1,5 @@ import asyncio +from inspect import iscoroutinefunction import random from collections import defaultdict from dataclasses import dataclass @@ -22,7 +23,7 @@ class UpdatedObject: UpdateStateAsyncCallable = Callable[[Any], Awaitable[None]] -class LongPollerAsyncClient: +class LongPollAsyncClient: """The asynchronous long polling client. Internally, it runs `await object_ref` in a `while True` loop. When a @@ -31,7 +32,7 @@ class LongPollerAsyncClient: the next poll. Args: - host_actor(ray.ActorHandle): handle to actor embedding LongPollerHost. + host_actor(ray.ActorHandle): handle to actor embedding LongPollHost. key_listeners(Dict[str, AsyncCallable]): a dictionary mapping keys to callbacks to be called on state update for the corresponding keys. """ @@ -40,6 +41,10 @@ class LongPollerAsyncClient: key_listeners: Dict[str, UpdateStateAsyncCallable]) -> None: self.host_actor = host_actor self.key_listeners = key_listeners + for callback in key_listeners.values(): + if not iscoroutinefunction(callback): + raise ValueError( + "Callbacks to async long poller must be 'async def'.") self.snapshot_ids: Dict[str, int] = { key: -1 @@ -56,34 +61,31 @@ class LongPollerAsyncClient: self.snapshot_ids) return object_ref - def _update(self, updates: Dict[str, UpdatedObject]): - for key, update in updates.items(): - self.object_snapshots[key] = update.object_snapshot - self.snapshot_ids[key] = update.snapshot_id - async def _do_long_poll(self): while True: try: updates: Dict[str, UpdatedObject] = await self._poll_once() - self._update(updates) - logger.debug(f"LongPollerClient received udpates: {updates}") - for key, updated_object in updates.items(): + logger.debug("LongPollClient received updates for keys: " + f"{list(updates.keys())}.") + for key, update in updates.items(): + self.object_snapshots[key] = update.object_snapshot + self.snapshot_ids[key] = update.snapshot_id # NOTE(simon): # This blocks the loop from doing another poll. Consider # use loop.create_task here or poll first then call the # callbacks. callback = self.key_listeners[key] - await callback(updated_object.object_snapshot) + await callback(update.object_snapshot) except ray.exceptions.RayActorError: # This can happen during shutdown where the controller is # intentionally killed, the client should just gracefully # exit. - logger.debug("LongPollerClient failed to connect to host. " + logger.debug("LongPollClient failed to connect to host. " "Shutting down.") break -class LongPollerHost: +class LongPollHost: """The server side object that manages long pulling requests. The desired use case is to embed this in an Ray actor. Client will be @@ -115,11 +117,10 @@ class LongPollerHost: immediately if the snapshot_ids are outdated, otherwise it will block until there's one updates. """ - # 1. Figure out which keys do we care about - watched_keys = set(self.snapshot_ids.keys()).intersection( - keys_to_snapshot_ids.keys()) - if len(watched_keys) == 0: - raise ValueError("Keys not found.") + watched_keys = keys_to_snapshot_ids.keys() + nonexistent_keys = set(watched_keys) - set(self.snapshot_ids.keys()) + if len(nonexistent_keys) > 0: + raise ValueError(f"Keys not found: {nonexistent_keys}.") # 2. If there are any outdated keys (by comparing snapshot ids) # return immediately. @@ -159,7 +160,7 @@ class LongPollerHost: def notify_changed(self, object_key: str, updated_object: Any): self.snapshot_ids[object_key] += 1 self.object_snapshots[object_key] = updated_object - logger.debug(f"LongPollerHost: {object_key} = {updated_object}") + logger.debug(f"LongPollHost: Notify change for key {object_key}.") if object_key in self.notifier_events: for event in self.notifier_events.pop(object_key): diff --git a/python/ray/serve/router.py b/python/ray/serve/router.py index 9808ad9c9..8276e6cd2 100644 --- a/python/ray/serve/router.py +++ b/python/ray/serve/router.py @@ -6,9 +6,10 @@ from typing import Any, DefaultDict, Dict, Iterable, List, Optional import ray from ray.actor import ActorHandle +from ray.serve.constants import LongPollKey from ray.serve.context import TaskContext from ray.serve.endpoint_policy import EndpointPolicy, RandomEndpointPolicy -from ray.serve.long_poll import LongPollerAsyncClient +from ray.serve.long_poll import LongPollAsyncClient from ray.serve.utils import logger from ray.util import metrics @@ -106,7 +107,8 @@ class ReplicaSet: ) >= self.max_concurrent_queries: # This replica is overloaded, try next one continue - logger.debug(f"Replica set assigned {query} to {replica}") + logger.debug(f"Assigned query {query.metadata.request_id} " + f"to replica {replica}.") ref = replica.handle_request.remote(query) self.in_flight_queries[replica].add(ref) return ref @@ -133,7 +135,8 @@ class ReplicaSet: """ assigned_ref = self._try_assign_replica(query) while assigned_ref is None: # Can't assign a replica right now. - logger.debug(f"Failed to assign a replica for query {query}") + logger.debug("Failed to assign a replica for " + f"query {query.metadata.request_id}") # Maybe there exists a free replica, we just need to refresh our # query tracker. num_finished = self._drain_completed_object_refs() @@ -141,7 +144,7 @@ class ReplicaSet: # config to be updated. if num_finished == 0: logger.debug( - f"All replicas are busy, waiting for a free replica.") + "All replicas are busy, waiting for a free replica.") await asyncio.wait( self._all_query_refs + [self.config_updated_event.wait()], return_when=asyncio.FIRST_COMPLETED) @@ -176,14 +179,14 @@ class Router: async def setup_in_async_loop(self): # NOTE(simon): Instead of performing initialization in __init__, - # We separated the init of LongPollerAsyncClient to this method because - # __init__ might be called in sync context. LongPollerAsyncClient + # We separated the init of LongPollAsyncClient to this method because + # __init__ might be called in sync context. LongPollAsyncClient # requires async context. - self.long_pull_client = LongPollerAsyncClient( + self.long_poll_client = LongPollAsyncClient( self.controller, { - "traffic_policies": self._update_traffic_policies, - "worker_handles": self._update_worker_handles, - "backend_configs": self._update_backend_configs, + LongPollKey.TRAFFIC_POLICIES: self._update_traffic_policies, + LongPollKey.REPLICA_HANDLES: self._update_replica_handles, + LongPollKey.BACKEND_CONFIGS: self._update_backend_configs, }) async def _update_traffic_policies(self, traffic_policies): @@ -194,8 +197,8 @@ class Router: event = self._pending_endpoints.pop(endpoint) event.set() - async def _update_worker_handles(self, worker_handles): - for backend_tag, replica_handles in worker_handles.items(): + async def _update_replica_handles(self, replica_handles): + for backend_tag, replica_handles in replica_handles.items(): self.backend_replicas[backend_tag].update_worker_replicas( replica_handles) diff --git a/python/ray/serve/scripts.py b/python/ray/serve/scripts.py new file mode 100644 index 000000000..8a0baa7c0 --- /dev/null +++ b/python/ray/serve/scripts.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +import click + +import ray +from ray import serve +from ray.serve.constants import DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT + + +@click.group( + help="[EXPERIMENTAL] CLI for managing Serve instances on a Ray cluster.") +@click.option( + "--address", + "-a", + default="auto", + required=False, + type=str, + help="Address of the running Ray cluster to connect to. " + "Defaults to \"auto\".") +def cli(address): + ray.init(address=address) + + +@cli.command(help="Start a detached Serve instance on the Ray cluster.") +@click.option( + "--http-host", + default=DEFAULT_HTTP_HOST, + required=False, + type=str, + help="Host for HTTP servers to listen on. " + f"Defaults to {DEFAULT_HTTP_HOST}.") +@click.option( + "--http-port", + default=DEFAULT_HTTP_PORT, + required=False, + type=int, + help="Port for HTTP servers to listen on. " + f"Defaults to {DEFAULT_HTTP_PORT}.") +def start(http_host, http_port): + serve.start(detached=True, http_host=http_host, http_port=http_port) + + +@cli.command(help="Shutdown the running Serve instance on the Ray cluster.") +def shutdown(): + serve.connect().shutdown() diff --git a/python/ray/serve/tests/conftest.py b/python/ray/serve/tests/conftest.py index 740d82271..76a67c190 100644 --- a/python/ray/serve/tests/conftest.py +++ b/python/ray/serve/tests/conftest.py @@ -7,6 +7,7 @@ import pytest import ray from ray import serve from ray.serve.config import BackendConfig +from ray.serve.constants import LongPollKey if os.environ.get("RAY_SERVE_INTENTIONALLY_CRASH", False) == 1: serve.controller._CRASH_AFTER_CHECKPOINT_PROBABILITY = 0.5 @@ -42,22 +43,22 @@ def mock_controller_with_name(): @ray.remote(num_cpus=0) class MockControllerActor: def __init__(self): - from ray.serve.long_poll import LongPollerHost - self.host = LongPollerHost() + from ray.serve.long_poll import LongPollHost + self.host = LongPollHost() self.backend_replicas = defaultdict(list) self.backend_configs = dict() self.clear() def clear(self): - self.host.notify_changed("worker_handles", {}) - self.host.notify_changed("traffic_policies", {}) - self.host.notify_changed("backend_configs", {}) + self.host.notify_changed(LongPollKey.REPLICA_HANDLES, {}) + self.host.notify_changed(LongPollKey.TRAFFIC_POLICIES, {}) + self.host.notify_changed(LongPollKey.BACKEND_CONFIGS, {}) async def listen_for_change(self, snapshot_ids): return await self.host.listen_for_change(snapshot_ids) def set_traffic(self, endpoint, traffic_policy): - self.host.notify_changed("traffic_policies", + self.host.notify_changed(LongPollKey.TRAFFIC_POLICIES, {endpoint: traffic_policy}) def add_new_replica(self, @@ -68,15 +69,17 @@ def mock_controller_with_name(): self.backend_configs[backend_tag] = backend_config self.host.notify_changed( - "worker_handles", + LongPollKey.REPLICA_HANDLES, self.backend_replicas, ) - self.host.notify_changed("backend_configs", self.backend_configs) + self.host.notify_changed(LongPollKey.BACKEND_CONFIGS, + self.backend_configs) def update_backend(self, backend_tag: str, backend_config: BackendConfig): self.backend_configs[backend_tag] = backend_config - self.host.notify_changed("backend_configs", self.backend_configs) + self.host.notify_changed(LongPollKey.BACKEND_CONFIGS, + self.backend_configs) name = f"MockController{random.randint(0,10e4)}" yield name, MockControllerActor.options(name=name).remote() diff --git a/python/ray/serve/tests/test_api.py b/python/ray/serve/tests/test_api.py index 5d2bc76c0..1f236f915 100644 --- a/python/ray/serve/tests/test_api.py +++ b/python/ray/serve/tests/test_api.py @@ -4,6 +4,7 @@ import time import os import pytest import requests +import starlette.responses import ray from ray import serve @@ -25,22 +26,6 @@ def test_e2e(serve_instance): client.create_endpoint( "endpoint", backend="echo:v1", route="/api", methods=["GET", "POST"]) - retry_count = 5 - timeout_sleep = 0.5 - while True: - try: - resp = requests.get( - "http://127.0.0.1:8000/-/routes", timeout=0.5).json() - assert resp == {"/api": ["endpoint", ["GET", "POST"]]} - break - except Exception as e: - time.sleep(timeout_sleep) - timeout_sleep *= 2 - retry_count -= 1 - if retry_count == 0: - assert False, ("Route table hasn't been updated after 3 tries." - "The latest error was {}").format(e) - resp = requests.get("http://127.0.0.1:8000/api").json()["method"] assert resp == "GET" @@ -48,6 +33,63 @@ def test_e2e(serve_instance): assert resp == "POST" +def test_starlette_response(serve_instance): + client = serve_instance + + def basic_response(_): + return starlette.responses.Response( + "Hello, world!", media_type="text/plain") + + client.create_backend("basic_response", basic_response) + client.create_endpoint( + "basic_response", backend="basic_response", route="/basic_response") + assert requests.get( + "http://127.0.0.1:8000/basic_response").text == "Hello, world!" + + def html_response(_): + return starlette.responses.HTMLResponse( + "

Hello, world!

") + + client.create_backend("html_response", html_response) + client.create_endpoint( + "html_response", backend="html_response", route="/html_response") + assert requests.get( + "http://127.0.0.1:8000/html_response" + ).text == "

Hello, world!

" + + def plain_text_response(_): + return starlette.responses.PlainTextResponse("Hello, world!") + + client.create_backend("plain_text_response", plain_text_response) + client.create_endpoint( + "plain_text_response", + backend="plain_text_response", + route="/plain_text_response") + assert requests.get( + "http://127.0.0.1:8000/plain_text_response").text == "Hello, world!" + + def json_response(_): + return starlette.responses.JSONResponse({"hello": "world"}) + + client.create_backend("json_response", json_response) + client.create_endpoint( + "json_response", backend="json_response", route="/json_response") + assert requests.get("http://127.0.0.1:8000/json_response").json()[ + "hello"] == "world" + + def redirect_response(_): + return starlette.responses.RedirectResponse( + url="http://127.0.0.1:8000/basic_response") + + client.create_backend("redirect_response", redirect_response) + client.create_endpoint( + "redirect_response", + backend="redirect_response", + route="/redirect_response") + assert requests.get( + "http://127.0.0.1:8000/redirect_response").text == "Hello, world!" + + def test_backend_user_config(serve_instance): client = serve_instance @@ -63,25 +105,26 @@ def test_backend_user_config(serve_instance): config = BackendConfig(num_replicas=2, user_config={"count": 123, "b": 2}) client.create_backend("counter", Counter, config=config) - client.create_endpoint("counter", backend="counter", route="/counter") + client.create_endpoint("counter", backend="counter") handle = client.get_handle("counter") def check(val, num_replicas): pids_seen = set() for i in range(100): result = ray.get(handle.remote()) - assert (str(result[0]) == val), result[0] + if str(result[0]) != val: + return False pids_seen.add(result[1]) - assert (len(pids_seen) == num_replicas) + return len(pids_seen) == num_replicas - check("123", 2) + wait_for_condition(lambda: check("123", 2)) client.update_backend_config("counter", BackendConfig(num_replicas=3)) - check("123", 3) + wait_for_condition(lambda: check("123", 3)) config = BackendConfig(user_config={"count": 456}) client.update_backend_config("counter", config) - check("456", 3) + wait_for_condition(lambda: check("456", 3)) def test_call_method(serve_instance): @@ -183,7 +226,7 @@ def test_reject_duplicate_endpoint_and_route(serve_instance): def test_no_http(serve_instance): client = serve.start(http_host=None) - assert len(ray.get(client._controller.get_routers.remote())) == 0 + assert len(ray.get(client._controller.get_http_proxies.remote())) == 0 def hello(*args): return "hello" @@ -223,11 +266,6 @@ def test_scaling_replicas(serve_instance): client.create_endpoint("counter", backend="counter:v1", route="/increment") - # Keep checking the routing table until /increment is populated - while "/increment" not in requests.get( - "http://127.0.0.1:8000/-/routes").json(): - time.sleep(0.2) - counter_result = [] for _ in range(10): resp = requests.get("http://127.0.0.1:8000/increment").json() @@ -267,11 +305,6 @@ def test_batching(serve_instance): client.create_endpoint( "counter1", backend="counter:v11", route="/increment2") - # Keep checking the routing table until /increment is populated - while "/increment2" not in requests.get( - "http://127.0.0.1:8000/-/routes").json(): - time.sleep(0.2) - future_list = [] handle = client.get_handle("counter1") for _ in range(20): @@ -299,8 +332,7 @@ def test_batching_exception(serve_instance): # Set the max batch size. config = BackendConfig(max_batch_size=5) client.create_backend("exception:v1", NoListReturned, config=config) - client.create_endpoint( - "exception-test", backend="exception:v1", route="/noListReturned") + client.create_endpoint("exception-test", backend="exception:v1") handle = client.get_handle("exception-test") with pytest.raises(ray.exceptions.RayTaskError): @@ -323,16 +355,16 @@ def test_updating_config(serve_instance): client.create_endpoint("bsimple", backend="bsimple:v1", route="/bsimple") controller = client._controller - old_replica_tag_list = ray.get( - controller._list_replicas.remote("bsimple:v1")) + old_replica_tag_list = list( + ray.get(controller._all_replica_handles.remote())["bsimple:v1"].keys()) update_config = BackendConfig(max_batch_size=5) client.update_backend_config("bsimple:v1", update_config) - new_replica_tag_list = ray.get( - controller._list_replicas.remote("bsimple:v1")) + new_replica_tag_list = list( + ray.get(controller._all_replica_handles.remote())["bsimple:v1"].keys()) new_all_tag_list = [] for worker_dict in ray.get( - controller.get_all_replica_handles.remote()).values(): + controller._all_replica_handles.remote()).values(): new_all_tag_list.extend(list(worker_dict.keys())) # the old and new replica tag list should be identical @@ -648,7 +680,7 @@ def test_create_infeasible_error(serve_instance): "MagicMLResource": 100 }}) - # Even each replica might be feasible, the total might not be. + # Even though each replica might be feasible, the total might not be. current_cpus = int(ray.nodes()[0]["Resources"]["CPU"]) num_replicas = current_cpus + 20 config = BackendConfig(num_replicas=num_replicas) @@ -661,10 +693,6 @@ def test_create_infeasible_error(serve_instance): }}, config=config) - # No replica should be created! - replicas = ray.get(client._controller._list_replicas.remote("f1")) - assert len(replicas) == 0 - def test_shutdown(): def f(): @@ -797,6 +825,7 @@ def test_serve_metrics(serve_instance): client.create_backend("metrics", batcher) client.create_endpoint("metrics", backend="metrics", route="/metrics") + # send 10 concurrent requests url = "http://127.0.0.1:8000/metrics" ray.get([block_until_http_ready.remote(url) for _ in range(10)]) diff --git a/python/ray/serve/tests/test_backend_worker.py b/python/ray/serve/tests/test_backend_worker.py index 4233740fa..1b03c0835 100644 --- a/python/ray/serve/tests/test_backend_worker.py +++ b/python/ray/serve/tests/test_backend_worker.py @@ -48,7 +48,7 @@ def setup_worker(name, async def add_servable_to_router(servable, router, controller_name, **kwargs): worker = setup_worker( "backend", servable, controller_name=controller_name, **kwargs) - await router._update_worker_handles.remote({"backend": [worker]}) + await router._update_replica_handles.remote({"backend": [worker]}) await router._update_traffic_policies.remote({ "endpoint": TrafficPolicy({ "backend": 1.0 diff --git a/python/ray/serve/tests/test_controller.py b/python/ray/serve/tests/test_controller.py new file mode 100644 index 000000000..1900d39dc --- /dev/null +++ b/python/ray/serve/tests/test_controller.py @@ -0,0 +1,23 @@ +import pytest + +import ray + + +def test_controller_inflight_requests_clear(serve_instance): + client = serve_instance + initial_number_reqs = ray.get( + client._controller._num_inflight_results.remote()) + + def function(_): + return "hello" + + client.create_backend("tst", function) + client.create_endpoint("end_pt", backend="tst") + + assert ray.get(client._controller._num_inflight_results.remote() + ) - initial_number_reqs == 0 + + +if __name__ == "__main__": + import sys + sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/ray/serve/tests/test_failure.py b/python/ray/serve/tests/test_failure.py index 99a05ca39..7e2fbcb65 100644 --- a/python/ray/serve/tests/test_failure.py +++ b/python/ray/serve/tests/test_failure.py @@ -4,6 +4,7 @@ import tempfile import time import ray +from ray.test_utils import wait_for_condition from ray import serve from ray.serve.config import BackendConfig, ReplicaConfig @@ -53,9 +54,11 @@ def test_controller_failure(serve_instance): client.create_backend("controller_failure:v2", function) client.set_traffic("controller_failure", {"controller_failure:v2": 1.0}) - for _ in range(10): + def check_controller_failure(): response = request_with_retries("/controller_failure", timeout=30) - assert response.text == "hello2" + return response.text == "hello2" + + wait_for_condition(check_controller_failure) def function(_): return "hello3" @@ -76,10 +79,10 @@ def test_controller_failure(serve_instance): assert response.text == "hello3" -def _kill_routers(client): - routers = ray.get(client._controller.get_routers.remote()) - for router in routers.values(): - ray.kill(router, no_restart=False) +def _kill_http_proxies(client): + http_proxies = ray.get(client._controller.get_http_proxies.remote()) + for http_proxy in http_proxies.values(): + ray.kill(http_proxy, no_restart=False) def test_http_proxy_failure(serve_instance): @@ -98,7 +101,7 @@ def test_http_proxy_failure(serve_instance): response = request_with_retries("/proxy_failure", timeout=30) assert response.text == "hello1" - _kill_routers(client) + _kill_http_proxies(client) def function(_): return "hello2" @@ -113,7 +116,7 @@ def test_http_proxy_failure(serve_instance): def _get_worker_handles(client, backend): controller = client._controller - backend_dict = ray.get(controller.get_all_replica_handles.remote()) + backend_dict = ray.get(controller._all_replica_handles.remote()) return list(backend_dict[backend].values()) @@ -124,7 +127,7 @@ def test_worker_restart(serve_instance): client = serve_instance class Worker1: - def __call__(self): + def __call__(self, *args): return os.getpid() client.create_backend("worker_failure:v1", Worker1) @@ -176,7 +179,7 @@ def test_worker_replica_failure(serve_instance): while True: pass - def __call__(self): + def __call__(self, *args): pass temp_path = os.path.join(tempfile.gettempdir(), diff --git a/python/ray/serve/tests/test_long_poll.py b/python/ray/serve/tests/test_long_poll.py index 040219527..5916eadcb 100644 --- a/python/ray/serve/tests/test_long_poll.py +++ b/python/ray/serve/tests/test_long_poll.py @@ -1,5 +1,4 @@ import sys -import functools import time import asyncio import os @@ -8,12 +7,12 @@ from typing import Dict import pytest import ray -from ray.serve.long_poll import (LongPollerAsyncClient, LongPollerHost, +from ray.serve.long_poll import (LongPollAsyncClient, LongPollHost, UpdatedObject) def test_host_standalone(serve_instance): - host = ray.remote(LongPollerHost).remote() + host = ray.remote(LongPollHost).remote() # Write two values ray.get(host.notify_changed.remote("key_1", 999)) @@ -44,10 +43,10 @@ def test_long_poll_restarts(serve_instance): max_restarts=-1, max_task_retries=-1, ) - class RestartableLongPollerHost: + class RestartableLongPollHost: def __init__(self) -> None: print("actor started") - self.host = LongPollerHost() + self.host = LongPollHost() self.host.notify_changed("timer", time.time()) self.should_exit = False @@ -63,7 +62,7 @@ def test_long_poll_restarts(serve_instance): print("actor exit") os._exit(1) - host = RestartableLongPollerHost.remote() + host = RestartableLongPollHost.remote() updated_values = ray.get(host.listen_for_change.remote({"timer": -1})) timer: UpdatedObject = updated_values["timer"] @@ -81,22 +80,31 @@ def test_long_poll_restarts(serve_instance): @pytest.mark.asyncio async def test_async_client(serve_instance): - host = ray.remote(LongPollerHost).remote() + host = ray.remote(LongPollHost).remote() # Write two values ray.get(host.notify_changed.remote("key_1", 100)) ray.get(host.notify_changed.remote("key_2", 999)) + # Check that construction fails with a sync callback. + def callback(result, key): + pass + + with pytest.raises(ValueError): + client = LongPollAsyncClient(host, {"key": callback}) + callback_results = dict() - async def callback(result, key): - callback_results[key] = result + async def key_1_callback(result): + callback_results["key_1"] = result - client = LongPollerAsyncClient( - host, { - "key_1": functools.partial(callback, key="key_1"), - "key_2": functools.partial(callback, key="key_2") - }) + async def key_2_callback(result): + callback_results["key_2"] = result + + client = LongPollAsyncClient(host, { + "key_1": key_1_callback, + "key_2": key_2_callback, + }) while len(client.object_snapshots) == 0: # Yield the loop for client to get the result diff --git a/python/ray/serve/utils.py b/python/ray/serve/utils.py index e9c95925d..efa6f1b6b 100644 --- a/python/ray/serve/utils.py +++ b/python/ray/serve/utils.py @@ -144,6 +144,7 @@ class ServeEncoder(json.JSONEncoder): @ray.remote(num_cpus=0) def block_until_http_ready(http_endpoint, backoff_time_s=1, + check_ready=None, timeout=HTTP_PROXY_TIMEOUT): http_is_ready = False start_time = time.time() @@ -152,7 +153,10 @@ def block_until_http_ready(http_endpoint, try: resp = requests.get(http_endpoint) assert resp.status_code == 200 - http_is_ready = True + if check_ready is None: + http_is_ready = True + else: + http_is_ready = check_ready(resp) except Exception: pass diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index ad9ab4ec0..573131fec 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -95,6 +95,8 @@ py_test_module_list( "test_dask_callback.py", "test_debug_tools.py", "test_experimental_client.py", + "test_experimental_client_metadata.py", + "test_experimental_client_terminate.py", "test_job.py", "test_memstat.py", "test_metrics_agent.py", diff --git a/python/ray/tests/project_files/session-tests/git-repo-pass/ray-project/requirements.txt b/python/ray/tests/project_files/session-tests/git-repo-pass/ray-project/requirements.txt index 0f026d879..e02756318 100644 --- a/python/ray/tests/project_files/session-tests/git-repo-pass/ray-project/requirements.txt +++ b/python/ray/tests/project_files/session-tests/git-repo-pass/ray-project/requirements.txt @@ -1 +1 @@ -ray[debug] \ No newline at end of file +ray diff --git a/python/ray/tests/project_files/session-tests/invalid-config-fail/ray-project/requirements.txt b/python/ray/tests/project_files/session-tests/invalid-config-fail/ray-project/requirements.txt index 0f026d879..e02756318 100644 --- a/python/ray/tests/project_files/session-tests/invalid-config-fail/ray-project/requirements.txt +++ b/python/ray/tests/project_files/session-tests/invalid-config-fail/ray-project/requirements.txt @@ -1 +1 @@ -ray[debug] \ No newline at end of file +ray diff --git a/python/ray/tests/project_files/session-tests/project-pass/ray-project/requirements.txt b/python/ray/tests/project_files/session-tests/project-pass/ray-project/requirements.txt index 0f026d879..e02756318 100644 --- a/python/ray/tests/project_files/session-tests/project-pass/ray-project/requirements.txt +++ b/python/ray/tests/project_files/session-tests/project-pass/ray-project/requirements.txt @@ -1 +1 @@ -ray[debug] \ No newline at end of file +ray diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 5d885715d..05d53c1b3 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -8,6 +8,7 @@ try: except ImportError: pytest_timeout = None import sys +import tempfile import datetime import ray @@ -867,5 +868,61 @@ def test_actor_creation_latency(ray_start_regular_shared): actor_create_time - start, end - start)) +@pytest.mark.parametrize( + "exit_condition", + [ + # "out_of_scope", TODO(edoakes): enable this once fixed. + "__ray_terminate__", + "ray.actor.exit_actor", + "ray.kill" + ]) +def test_atexit_handler(ray_start_regular_shared, exit_condition): + @ray.remote + class A(): + def __init__(self, tmpfile, data): + import atexit + + def f(*args, **kwargs): + with open(tmpfile, "w") as f: + f.write(data) + f.flush() + + atexit.register(f) + + def ready(self): + pass + + def exit(self): + ray.actor.exit_actor() + + data = "hello" + tmpfile = tempfile.NamedTemporaryFile() + a = A.remote(tmpfile.name, data) + ray.get(a.ready.remote()) + + if exit_condition == "out_of_scope": + del a + elif exit_condition == "__ray_terminate__": + ray.wait([a.__ray_terminate__.remote()]) + elif exit_condition == "ray.actor.exit_actor": + ray.wait([a.exit.remote()]) + elif exit_condition == "ray.kill": + ray.kill(a) + else: + assert False, "Unrecognized condition" + + def check_file_written(): + with open(tmpfile.name) as f: + if f.read() == data: + return True + return False + + # ray.kill() should not trigger atexit handlers, all other methods should. + if exit_condition == "ray.kill": + assert not check_file_written() + else: + ray.test_utils.wait_for_condition(check_file_written) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tests/test_actor_advanced.py b/python/ray/tests/test_actor_advanced.py index af490eb77..8bc6801f8 100644 --- a/python/ray/tests/test_actor_advanced.py +++ b/python/ray/tests/test_actor_advanced.py @@ -1055,11 +1055,11 @@ def test_actor_resource_demand(shutdown_only): ray.get(a.foo.remote()) time.sleep(1) - message = global_state_accessor.get_all_heartbeat() - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString(message) + message = global_state_accessor.get_all_resource_usage() + resource_usages = ray.gcs_utils.ResourceUsageBatchData.FromString(message) # The actor is scheduled so there should be no more demands left. - assert len(heartbeat.resource_load_by_shape.resource_demands) == 0 + assert len(resource_usages.resource_load_by_shape.resource_demands) == 0 @ray.remote(num_cpus=80) class Actor2: @@ -1070,23 +1070,24 @@ def test_actor_resource_demand(shutdown_only): time.sleep(1) # This actor cannot be scheduled. - message = global_state_accessor.get_all_heartbeat() - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString(message) - assert len(heartbeat.resource_load_by_shape.resource_demands) == 1 - assert (heartbeat.resource_load_by_shape.resource_demands[0].shape == { - "CPU": 80.0 - }) - assert (heartbeat.resource_load_by_shape.resource_demands[0] + message = global_state_accessor.get_all_resource_usage() + resource_usages = ray.gcs_utils.ResourceUsageBatchData.FromString(message) + assert len(resource_usages.resource_load_by_shape.resource_demands) == 1 + assert ( + resource_usages.resource_load_by_shape.resource_demands[0].shape == { + "CPU": 80.0 + }) + assert (resource_usages.resource_load_by_shape.resource_demands[0] .num_infeasible_requests_queued == 1) actors.append(Actor2.remote()) time.sleep(1) # Two actors cannot be scheduled. - message = global_state_accessor.get_all_heartbeat() - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString(message) - assert len(heartbeat.resource_load_by_shape.resource_demands) == 1 - assert (heartbeat.resource_load_by_shape.resource_demands[0] + message = global_state_accessor.get_all_resource_usage() + resource_usages = ray.gcs_utils.ResourceUsageBatchData.FromString(message) + assert len(resource_usages.resource_load_by_shape.resource_demands) == 1 + assert (resource_usages.resource_load_by_shape.resource_demands[0] .num_infeasible_requests_queued == 2) global_state_accessor.disconnect() diff --git a/python/ray/tests/test_actor_failures.py b/python/ray/tests/test_actor_failures.py index a6b9c381b..227fb48d2 100644 --- a/python/ray/tests/test_actor_failures.py +++ b/python/ray/tests/test_actor_failures.py @@ -1,3 +1,4 @@ +import asyncio import collections import numpy as np import os @@ -211,6 +212,66 @@ def test_actor_restart_with_retry(ray_init_with_task_retry_delay): ray.get(actor.increase.remote()) +def test_named_actor_max_task_retries(ray_init_with_task_retry_delay): + @ray.remote(num_cpus=0) + class Counter: + def __init__(self): + self.count = 0 + self.event = asyncio.Event() + + def increment(self): + self.count += 1 + self.event.set() + + async def wait_for_count(self, count): + while True: + if self.count >= count: + return + await self.event.wait() + self.event.clear() + + @ray.remote + class ActorToKill: + def __init__(self, counter): + counter.increment.remote() + + def run(self, counter, signal): + counter.increment.remote() + ray.get(signal.wait.remote()) + + @ray.remote + class CallingActor: + def __init__(self): + self.actor = ray.get_actor("a") + + def call_other(self, counter, signal): + return ray.get(self.actor.run.remote(counter, signal)) + + init_counter = Counter.remote() + run_counter = Counter.remote() + signal = SignalActor.remote() + + # Start the two actors, wait for ActorToKill's constructor to run. + a = ActorToKill.options( + name="a", max_restarts=-1, max_task_retries=-1).remote(init_counter) + c = CallingActor.remote() + ray.get(init_counter.wait_for_count.remote(1), timeout=30) + + # Signal the CallingActor to call ActorToKill, wait for it to be running, + # then kill ActorToKill. + # Verify that this causes ActorToKill's constructor to run a second time + # and the run method to begin a second time. + ref = c.call_other.remote(run_counter, signal) + ray.get(run_counter.wait_for_count.remote(1), timeout=30) + ray.kill(a, no_restart=False) + ray.get(init_counter.wait_for_count.remote(2), timeout=30) + ray.get(run_counter.wait_for_count.remote(2), timeout=30) + + # Signal the run method to finish, verify that the CallingActor returns. + signal.send.remote() + ray.get(ref, timeout=30) + + def test_actor_restart_on_node_failure(ray_start_cluster): config = { "num_heartbeats_timeout": 10, diff --git a/python/ray/tests/test_advanced_3.py b/python/ray/tests/test_advanced_3.py index 420c0c594..7f1e8e639 100644 --- a/python/ray/tests/test_advanced_3.py +++ b/python/ray/tests/test_advanced_3.py @@ -94,8 +94,13 @@ def test_local_scheduling_first(ray_start_cluster): assert local() -@pytest.mark.skipif(new_scheduler_enabled(), reason="flakes more often") -def test_load_balancing_with_dependencies(ray_start_cluster): +@pytest.mark.parametrize("fast", [True, False]) +def test_load_balancing_with_dependencies(ray_start_cluster, fast): + if fast and new_scheduler_enabled: + # Load-balancing on new scheduler can be inefficient if (task + # duration:heartbeat interval) is small enough. + pytest.skip() + # This test ensures that tasks are being assigned to all raylets in a # roughly equal manner even when the tasks have dependencies. cluster = ray_start_cluster @@ -106,7 +111,10 @@ def test_load_balancing_with_dependencies(ray_start_cluster): @ray.remote def f(x): - time.sleep(0.010) + if fast: + time.sleep(0.010) + else: + time.sleep(0.1) return ray.worker.global_worker.node.unique_id # This object will be local to one of the raylets. Make sure diff --git a/python/ray/tests/test_asyncio.py b/python/ray/tests/test_asyncio.py index 9bf3d86b1..18dd63a22 100644 --- a/python/ray/tests/test_asyncio.py +++ b/python/ray/tests/test_asyncio.py @@ -198,6 +198,32 @@ async def test_asyncio_double_await(ray_start_regular_shared): await waiting +@pytest.mark.asyncio +async def test_asyncio_exit_actor(ray_start_regular_shared): + # https://github.com/ray-project/ray/issues/12649 + # The test should just hang without the fix. + + @ray.remote + class Actor: + async def exit(self): + ray.actor.exit_actor() + + async def ping(self): + return "pong" + + async def loop_forever(self): + while True: + await asyncio.sleep(5) + + a = Actor.options(max_task_retries=0).remote() + a.loop_forever.remote() + # Make sure exit_actor exits immediately, not once all tasks completed. + ray.get(a.exit.remote()) + + with pytest.raises(ray.exceptions.RayActorError): + ray.get(a.ping.remote()) + + if __name__ == "__main__": import pytest sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index f99b3791c..fa9b9241e 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -537,6 +537,7 @@ class AutoscalingTest(unittest.TestCase): self.provider = MockProvider() self.provider.create_node({}, {TAG_RAY_NODE_KIND: "worker"}, 10) runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(10)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -558,6 +559,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(11)]) lm = LoadMetrics() autoscaler = StandardAutoscaler( config_path, @@ -613,6 +615,70 @@ class AutoscalingTest(unittest.TestCase): autoscaler.update() self.waitForNodes(0) + def testLegacyYamlWithRequestResources(self): + """Test when using legacy yamls request_resources() adds workers. + + Makes sure that requested resources are added for legacy yamls when + necessary. So if requested resources for instance fit on the headnode + we don't add more nodes. But we add more nodes when they don't fit. + """ + config = SMALL_CLUSTER.copy() + config["min_workers"] = 0 + config["max_workers"] = 100 + config["idle_timeout_minutes"] = 0 + config["upscaling_speed"] = 1 + config_path = self.write_config(config) + + self.provider = MockProvider() + self.provider.create_node({}, { + TAG_RAY_NODE_KIND: NODE_KIND_HEAD, + TAG_RAY_USER_NODE_TYPE: NODE_TYPE_LEGACY_HEAD + }, 1) + head_ip = self.provider.non_terminated_node_ips( + tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_HEAD}, )[0] + runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(10)]) + + lm = LoadMetrics() + lm.local_ip = head_ip + lm.update(head_ip, {"CPU": 1}, {"CPU": 1}, {}) + autoscaler = StandardAutoscaler( + config_path, + lm, + max_launch_batch=5, + max_concurrent_launches=5, + max_failures=0, + process_runner=runner, + update_interval_s=0) + autoscaler.update() + # 1 head node. + self.waitForNodes(1) + autoscaler.request_resources([{"CPU": 1}]) + autoscaler.update() + # still 1 head node because request_resources fits in the headnode. + self.waitForNodes(1) + autoscaler.request_resources([{"CPU": 1}] + [{"CPU": 2}] * 9) + autoscaler.update() + self.waitForNodes(2) # Adds a single worker to get its resources. + autoscaler.update() + self.waitForNodes(2) # Still 1 worker because its resources + # aren't known. + lm.update("172.0.0.1", {"CPU": 2}, {"CPU": 2}, {}) + autoscaler.update() + self.waitForNodes(10) # 9 workers and 1 head node, scaled immediately. + lm.update( + "172.0.0.1", {"CPU": 2}, {"CPU": 2}, {}, + waiting_bundles=[{ + "CPU": 2 + }] * 9, + infeasible_bundles=[{ + "CPU": 1 + }] * 1) + autoscaler.update() + # Make sure that if all the resources fit on the exising nodes not + # to add any more. + self.waitForNodes(10) + def testAggressiveAutoscaling(self): config = SMALL_CLUSTER.copy() config["min_workers"] = 0 @@ -629,7 +695,7 @@ class AutoscalingTest(unittest.TestCase): head_ip = self.provider.non_terminated_node_ips( tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_HEAD}, )[0] runner = MockProcessRunner() - + runner.respond_to_call("json .Config.Env", ["[]" for i in range(11)]) lm = LoadMetrics() lm.local_ip = head_ip @@ -782,6 +848,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -817,6 +884,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(config) self.provider = MockProvider() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(10)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -896,6 +964,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(10)]) lm = LoadMetrics() autoscaler = StandardAutoscaler( config_path, @@ -949,6 +1018,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(4)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -989,6 +1059,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(config) self.provider = MockProvider() runner = MockProcessRunner(fail_cmds=["setup_cmd"]) + runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -1000,14 +1071,18 @@ class AutoscalingTest(unittest.TestCase): self.waitForNodes(2) self.provider.finish_starting_nodes() autoscaler.update() - self.waitForNodes( - 2, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UPDATE_FAILED}) + try: + self.waitForNodes( + 2, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UPDATE_FAILED}) + except AssertionError: + # The failed nodes might have been already terminated by autoscaler + assert len(self.provider.non_terminated_nodes({})) == 0 def testConfiguresOutdatedNodes(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() - runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + runner.respond_to_call("json .Config.Env", ["[]" for i in range(4)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -1038,6 +1113,7 @@ class AutoscalingTest(unittest.TestCase): self.provider = MockProvider() lm = LoadMetrics() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(5)]) autoscaler = StandardAutoscaler( config_path, lm, @@ -1087,12 +1163,22 @@ class AutoscalingTest(unittest.TestCase): autoscaler.update() assert autoscaler.pending_launches.value == 0 - assert len(self.provider.non_terminated_nodes({})) == 3 + # This actually remained 4 instead of 3, because the other 2 nodes + # are not connected and hence we rely more on connected nodes for + # min_workers. When the "pending" nodes show up as connected, + # then we can terminate the ones connected before. + assert len(self.provider.non_terminated_nodes({})) == 4 lm.last_used_time_by_ip["172.0.0.2"] = 0 lm.last_used_time_by_ip["172.0.0.3"] = 0 autoscaler.update() assert autoscaler.pending_launches.value == 0 - assert len(self.provider.non_terminated_nodes({})) == 1 + # 2 nodes and not 1 because 1 is needed for min_worker and the other 1 + # is still not connected. + self.waitForNodes(2) + # when we connect it, we will see 1 node. + lm.last_used_time_by_ip["172.0.0.4"] = 0 + autoscaler.update() + self.waitForNodes(1) def testTargetUtilizationFraction(self): config = SMALL_CLUSTER.copy() @@ -1103,6 +1189,7 @@ class AutoscalingTest(unittest.TestCase): self.provider = MockProvider() lm = LoadMetrics() runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(12)]) autoscaler = StandardAutoscaler( config_path, lm, @@ -1161,7 +1248,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() - runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + runner.respond_to_call("json .Config.Env", ["[]" for i in range(3)]) lm = LoadMetrics() autoscaler = StandardAutoscaler( config_path, diff --git a/python/ray/tests/test_cross_language.py b/python/ray/tests/test_cross_language.py index 3904f63df..10766b18b 100644 --- a/python/ray/tests/test_cross_language.py +++ b/python/ray/tests/test_cross_language.py @@ -1,4 +1,5 @@ import pytest +import sys import ray import ray.cluster_utils @@ -6,7 +7,7 @@ import ray.test_utils def test_cross_language_raise_kwargs(shutdown_only): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) with pytest.raises(Exception, match="kwargs"): ray.java_function("a", "b").remote(x="arg1") @@ -16,7 +17,7 @@ def test_cross_language_raise_kwargs(shutdown_only): def test_cross_language_raise_exception(shutdown_only): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) class PythonObject(object): pass diff --git a/python/ray/tests/test_experimental_client.py b/python/ray/tests/test_experimental_client.py index 8fc07590e..cbe52675f 100644 --- a/python/ray/tests/test_experimental_client.py +++ b/python/ray/tests/test_experimental_client.py @@ -2,7 +2,7 @@ import pytest from contextlib import contextmanager import ray.experimental.client.server.server as ray_client_server -from ray.experimental.client import ray +from ray.experimental.client import ray, reset_api from ray.experimental.client.common import ClientObjectRef @@ -10,9 +10,12 @@ from ray.experimental.client.common import ClientObjectRef def ray_start_client_server(): server = ray_client_server.serve("localhost:50051", test_mode=True) ray.connect("localhost:50051") - yield ray - ray.disconnect() - server.stop(0) + try: + yield ray + finally: + ray.disconnect() + server.stop(0) + reset_api() def test_real_ray_fallback(ray_start_regular_shared): @@ -34,9 +37,6 @@ def test_real_ray_fallback(ray_start_regular_shared): nodes = ray.get(get_nodes.remote()) assert len(nodes) == 1, nodes - with pytest.raises(NotImplementedError): - print(ray.nodes()) - def test_nested_function(ray_start_regular_shared): with ray_start_client_server() as ray: @@ -170,6 +170,70 @@ def test_basic_actor(ray_start_regular_shared): assert count == 2 +def test_pass_handles(ray_start_regular_shared): + """ + Test that passing client handles to actors and functions to remote actors + in functions (on the server or raylet side) works transparently to the + caller. + """ + with ray_start_client_server() as ray: + + @ray.remote + class ExecActor: + def exec(self, f, x): + return ray.get(f.remote(x)) + + def exec_exec(self, actor, f, x): + return ray.get(actor.exec.remote(f, x)) + + @ray.remote + def fact(x): + out = 1 + while x > 0: + out = out * x + x -= 1 + return out + + @ray.remote + def func_exec(f, x): + return ray.get(f.remote(x)) + + @ray.remote + def func_actor_exec(actor, f, x): + return ray.get(actor.exec.remote(f, x)) + + @ray.remote + def sneaky_func_exec(obj, x): + return ray.get(obj["f"].remote(x)) + + @ray.remote + def sneaky_actor_exec(obj, x): + return ray.get(obj["actor"].exec.remote(obj["f"], x)) + + def local_fact(x): + if x <= 0: + return 1 + return x * local_fact(x - 1) + + assert ray.get(fact.remote(7)) == local_fact(7) + assert ray.get(func_exec.remote(fact, 8)) == local_fact(8) + test_obj = {} + test_obj["f"] = fact + assert ray.get(sneaky_func_exec.remote(test_obj, 5)) == local_fact(5) + actor_handle = ExecActor.remote() + assert ray.get(actor_handle.exec.remote(fact, 7)) == local_fact(7) + assert ray.get(func_actor_exec.remote(actor_handle, fact, + 10)) == local_fact(10) + second_actor = ExecActor.remote() + assert ray.get(actor_handle.exec_exec.remote(second_actor, fact, + 9)) == local_fact(9) + test_actor_obj = {} + test_actor_obj["actor"] = second_actor + test_actor_obj["f"] = fact + assert ray.get(sneaky_actor_exec.remote(test_actor_obj, + 4)) == local_fact(4) + + if __name__ == "__main__": import sys sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tests/test_experimental_client_metadata.py b/python/ray/tests/test_experimental_client_metadata.py new file mode 100644 index 000000000..d0bb86c9e --- /dev/null +++ b/python/ray/tests/test_experimental_client_metadata.py @@ -0,0 +1,25 @@ +from ray.tests.test_experimental_client import ray_start_client_server + + +def test_get_ray_metadata(ray_start_regular_shared): + """ + Test the ClusterInfo client data pathway and API surface + """ + with ray_start_client_server() as ray: + ip_address = ray_start_regular_shared["node_ip_address"] + + initialized = ray.is_initialized() + assert initialized + + nodes = ray.nodes() + assert len(nodes) == 1, nodes + assert nodes[0]["NodeManagerAddress"] == ip_address + + current_node_id = "node:" + ip_address + + cluster_resources = ray.cluster_resources() + available_resources = ray.available_resources() + + assert cluster_resources["CPU"] == 1.0 + assert current_node_id in cluster_resources + assert current_node_id in available_resources diff --git a/python/ray/tests/test_experimental_client_terminate.py b/python/ray/tests/test_experimental_client_terminate.py new file mode 100644 index 000000000..e44c617e6 --- /dev/null +++ b/python/ray/tests/test_experimental_client_terminate.py @@ -0,0 +1,99 @@ +import pytest +import asyncio +from ray.tests.test_experimental_client import ray_start_client_server +from ray.test_utils import wait_for_condition +from ray.exceptions import TaskCancelledError +from ray.exceptions import RayTaskError +from ray.exceptions import WorkerCrashedError +from ray.exceptions import ObjectLostError +from ray.exceptions import GetTimeoutError + + +def valid_exceptions(use_force): + if use_force: + return (RayTaskError, TaskCancelledError, WorkerCrashedError, + ObjectLostError) + else: + return (RayTaskError, TaskCancelledError) + + +def _all_actors_dead(ray): + import ray as real_ray + + def _all_actors_dead_internal(): + return all(actor["State"] == real_ray.gcs_utils.ActorTableData.DEAD + for actor in list(real_ray.actors().values())) + + return _all_actors_dead_internal + + +def test_kill_actor_immediately_after_creation(ray_start_regular): + with ray_start_client_server() as ray: + + @ray.remote + class A: + pass + + a = A.remote() + b = A.remote() + + ray.kill(a) + ray.kill(b) + wait_for_condition(_all_actors_dead(ray), timeout=10) + + +@pytest.mark.parametrize("use_force", [True, False]) +def test_cancel_chain(ray_start_regular, use_force): + with ray_start_client_server() as ray: + + @ray.remote + class SignalActor: + def __init__(self): + self.ready_event = asyncio.Event() + + def send(self, clear=False): + self.ready_event.set() + if clear: + self.ready_event.clear() + + async def wait(self, should_wait=True): + if should_wait: + await self.ready_event.wait() + + signaler = SignalActor.remote() + + @ray.remote + def wait_for(t): + return ray.get(t[0]) + + obj1 = wait_for.remote([signaler.wait.remote()]) + obj2 = wait_for.remote([obj1]) + obj3 = wait_for.remote([obj2]) + obj4 = wait_for.remote([obj3]) + + assert len(ray.wait([obj1], timeout=.1)[0]) == 0 + ray.cancel(obj1, force=use_force) + for ob in [obj1, obj2, obj3, obj4]: + with pytest.raises(valid_exceptions(use_force)): + ray.get(ob) + + signaler2 = SignalActor.remote() + obj1 = wait_for.remote([signaler2.wait.remote()]) + obj2 = wait_for.remote([obj1]) + obj3 = wait_for.remote([obj2]) + obj4 = wait_for.remote([obj3]) + + assert len(ray.wait([obj3], timeout=.1)[0]) == 0 + ray.cancel(obj3, force=use_force) + for ob in [obj3, obj4]: + with pytest.raises(valid_exceptions(use_force)): + ray.get(ob) + + with pytest.raises(GetTimeoutError): + ray.get(obj1, timeout=.1) + + with pytest.raises(GetTimeoutError): + ray.get(obj2, timeout=.1) + + signaler2.send.remote() + ray.get(obj1) diff --git a/python/ray/tests/test_global_gc.py b/python/ray/tests/test_global_gc.py index 685edacf0..247516450 100644 --- a/python/ray/tests/test_global_gc.py +++ b/python/ray/tests/test_global_gc.py @@ -15,6 +15,43 @@ from ray.internal.internal_api import global_gc logger = logging.getLogger(__name__) +def test_auto_local_gc(shutdown_only): + ray.init(num_cpus=2, _system_config={"local_gc_interval_s": 1}) + + class ObjectWithCyclicRef: + def __init__(self): + self.loop = self + + @ray.remote(num_cpus=1) + class GarbageHolder: + def __init__(self): + gc.disable() + x = ObjectWithCyclicRef() + self.garbage = weakref.ref(x) + + def has_garbage(self): + return self.garbage() is not None + + try: + gc.disable() + + # Local driver. + local_ref = weakref.ref(ObjectWithCyclicRef()) + + # Remote workers. + actors = [GarbageHolder.remote() for _ in range(2)] + assert local_ref() is not None + assert all(ray.get([a.has_garbage.remote() for a in actors])) + + def check_refs_gced(): + return (local_ref() is None and + not any(ray.get([a.has_garbage.remote() for a in actors]))) + + wait_for_condition(check_refs_gced) + finally: + gc.enable() + + def test_global_gc(shutdown_only): cluster = ray.cluster_utils.Cluster() for _ in range(2): diff --git a/python/ray/tests/test_global_state.py b/python/ray/tests/test_global_state.py index 967d6d7ea..c201b6bc3 100644 --- a/python/ray/tests/test_global_state.py +++ b/python/ray/tests/test_global_state.py @@ -144,7 +144,6 @@ def test_global_state_actor_entry(ray_start_regular): @pytest.mark.parametrize("max_shapes", [0, 2, -1]) -@pytest.mark.skipif(new_scheduler_enabled(), reason="broken") def test_load_report(shutdown_only, max_shapes): resource1 = "A" resource2 = "B" @@ -173,13 +172,14 @@ def test_load_report(shutdown_only, max_shapes): self.report = None def check_load_report(self): - message = global_state_accessor.get_all_heartbeat() + message = global_state_accessor.get_all_resource_usage() if message is None: return False - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString( + resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString( message) - self.report = heartbeat.resource_load_by_shape.resource_demands + self.report = \ + resource_usage.resource_load_by_shape.resource_demands if max_shapes == 0: return True elif max_shapes == 2: @@ -195,6 +195,8 @@ def test_load_report(shutdown_only, max_shapes): if max_shapes != -1: assert len(checker.report) <= max_shapes + print(checker.report) + if max_shapes > 0: # Check that we always include the 1-CPU resource shape. one_cpu_shape = {"CPU": 1} @@ -215,7 +217,8 @@ def test_load_report(shutdown_only, max_shapes): global_state_accessor.disconnect() -@pytest.mark.skipif(new_scheduler_enabled(), reason="broken") +@pytest.mark.skipif( + new_scheduler_enabled(), reason="requires placement groups") def test_placement_group_load_report(ray_start_cluster): cluster = ray_start_cluster # Add a head node that doesn't have gpu resource. @@ -227,40 +230,40 @@ def test_placement_group_load_report(ray_start_cluster): class PgLoadChecker: def nothing_is_ready(self): - heartbeat = self._read_heartbeat() - if not heartbeat: + resource_usage = self._read_resource_usage() + if not resource_usage: return False - if heartbeat.HasField("placement_group_load"): - pg_load = heartbeat.placement_group_load + if resource_usage.HasField("placement_group_load"): + pg_load = resource_usage.placement_group_load return len(pg_load.placement_group_data) == 2 return False def only_first_one_ready(self): - heartbeat = self._read_heartbeat() - if not heartbeat: + resource_usage = self._read_resource_usage() + if not resource_usage: return False - if heartbeat.HasField("placement_group_load"): - pg_load = heartbeat.placement_group_load + if resource_usage.HasField("placement_group_load"): + pg_load = resource_usage.placement_group_load return len(pg_load.placement_group_data) == 1 return False def two_infeasible_pg(self): - heartbeat = self._read_heartbeat() - if not heartbeat: + resource_usage = self._read_resource_usage() + if not resource_usage: return False - if heartbeat.HasField("placement_group_load"): - pg_load = heartbeat.placement_group_load + if resource_usage.HasField("placement_group_load"): + pg_load = resource_usage.placement_group_load return len(pg_load.placement_group_data) == 2 return False - def _read_heartbeat(self): - message = global_state_accessor.get_all_heartbeat() + def _read_resource_usage(self): + message = global_state_accessor.get_all_resource_usage() if message is None: return False - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString( + resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString( message) - return heartbeat + return resource_usage checker = PgLoadChecker() @@ -284,7 +287,6 @@ def test_placement_group_load_report(ray_start_cluster): global_state_accessor.disconnect() -@pytest.mark.skipif(new_scheduler_enabled(), reason="broken") def test_backlog_report(shutdown_only): cluster = ray.init( num_cpus=1, _system_config={ @@ -301,13 +303,14 @@ def test_backlog_report(shutdown_only): return None def backlog_size_set(): - message = global_state_accessor.get_all_heartbeat() + message = global_state_accessor.get_all_resource_usage() if message is None: return False - heartbeat = ray.gcs_utils.HeartbeatBatchTableData.FromString(message) + resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString( + message) aggregate_resource_load = \ - heartbeat.resource_load_by_shape.resource_demands + resource_usage.resource_load_by_shape.resource_demands if len(aggregate_resource_load) == 1: backlog_size = aggregate_resource_load[0].backlog_size print(backlog_size) diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 9386e4f1f..12490dd44 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -2,6 +2,7 @@ import json import pathlib import platform from pprint import pformat +import time from unittest.mock import MagicMock import requests @@ -293,6 +294,33 @@ def test_custom_metrics_edge_cases(metric_mock): Count("name", tag_keys=("a")) +def test_metrics_override_shouldnt_warn(ray_start_regular, log_pubsub): + # https://github.com/ray-project/ray/issues/12859 + + @ray.remote + def override(): + a = Count("num_count", description="") + b = Count("num_count", description="") + a.record(1) + b.record(1) + + ray.get(override.remote()) + + # Check the stderr from the worker. + start = time.time() + while True: + if (time.time() - start) > 5: + break + msg = log_pubsub.get_message() + if msg is None: + time.sleep(0.01) + continue + + log_lines = json.loads(ray.utils.decode(msg["data"]))["lines"] + for line in log_lines: + assert "Attempt to register measure" not in line + + if __name__ == "__main__": import sys sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tests/test_multi_tenancy.py b/python/ray/tests/test_multi_tenancy.py index b7caa8c08..856584558 100644 --- a/python/ray/tests/test_multi_tenancy.py +++ b/python/ray/tests/test_multi_tenancy.py @@ -10,9 +10,8 @@ import ray import ray.test_utils from ray.core.generated import common_pb2 from ray.core.generated import node_manager_pb2, node_manager_pb2_grpc -from ray.test_utils import ( - wait_for_condition, wait_for_pid_to_exit, run_string_as_driver, - run_string_as_driver_nonblocking, new_scheduler_enabled) +from ray.test_utils import (wait_for_condition, run_string_as_driver, + run_string_as_driver_nonblocking) def get_workers(): @@ -207,50 +206,6 @@ def test_worker_capping_run_chained_tasks(shutdown_only): assert len(get_workers()) == 2 -@pytest.mark.skipif(new_scheduler_enabled(), reason="fails") -def test_worker_capping_fifo(shutdown_only): - # Start 2 initial workers by setting num_cpus to 2. - info = ray.init(num_cpus=2) - wait_for_condition(lambda: len(get_workers()) == 2) - - time.sleep(1) - - @ray.remote - def getpid(): - return os.getpid() - - worker1, worker2 = get_workers() - - if worker1.pid == ray.get(getpid.remote()): - worker1, worker2 = [worker2, worker1] - - # Worker 1 is before worker 2 in the FIFO queue. - - driver_code = """ -import ray -import time - -ray.init(address="{}") - -@ray.remote -def foo(): - pass - -ray.get(foo.remote()) -# Sleep a while to make sure an idle worker exits before this driver exits. -time.sleep(2) -ray.shutdown() - """.format(info["redis_address"]) - - run_string_as_driver(driver_code) - - # Worker 1 should have been killed. - wait_for_pid_to_exit(worker1.pid) - - wait_for_condition(lambda: len(get_workers()) == 1) - assert worker2.pid == get_workers()[0].pid - - def test_worker_registration_failure_after_driver_exit(shutdown_only): info = ray.init(num_cpus=1) diff --git a/python/ray/tests/test_object_spilling.py b/python/ray/tests/test_object_spilling.py index 5a71f5166..9004fd030 100644 --- a/python/ray/tests/test_object_spilling.py +++ b/python/ray/tests/test_object_spilling.py @@ -12,7 +12,7 @@ import psutil import ray from ray.external_storage import (create_url_with_offset, parse_url_with_offset) -from ray.test_utils import new_scheduler_enabled, wait_for_condition +from ray.test_utils import wait_for_condition bucket_name = "object-spilling-test" spill_local_path = "/tmp/spill" @@ -338,7 +338,6 @@ def test_spill_objects_automatically(object_spilling_config, shutdown_only): @pytest.mark.skipif( platform.system() == "Windows", reason="Failing on Windows.") -@pytest.mark.skipif(new_scheduler_enabled(), reason="hangs") def test_spill_during_get(object_spilling_config, shutdown_only): ray.init( num_cpus=4, @@ -569,7 +568,6 @@ def test_delete_objects_on_worker_failure(tmp_path, shutdown_only): @pytest.mark.skipif( platform.system() == "Windows", reason="Failing on Windows.") -@pytest.mark.skipif(new_scheduler_enabled(), reason="flaky") def test_delete_objects_multi_node(tmp_path, ray_start_cluster): # Limit our object store to 75 MiB of memory. temp_folder = tmp_path / "spill" diff --git a/python/ray/tests/test_placement_group.py b/python/ray/tests/test_placement_group.py index 5acd0fc9e..066ff0e75 100644 --- a/python/ray/tests/test_placement_group.py +++ b/python/ray/tests/test_placement_group.py @@ -1,6 +1,7 @@ import pytest import os import sys +import time try: import pytest_timeout @@ -674,12 +675,18 @@ def test_atomic_creation(ray_start_cluster): @ray.remote(num_cpus=3) def bothering_task(): - import time - time.sleep(1) + time.sleep(6) return True # Schedule tasks to fail initial placement group creation. tasks = [bothering_task.remote() for _ in range(2)] + + # Make sure the two common task has scheduled. + def tasks_scheduled(): + return ray.available_resources()["CPU"] == 2.0 + + wait_for_condition(tasks_scheduled) + # Create an actor that will fail bundle scheduling. # It is important to use pack strategy to make test less flaky. pg = ray.util.placement_group( @@ -699,7 +706,7 @@ def test_atomic_creation(ray_start_cluster): # Wait on the placement group now. It should be unready # because normal actor takes resources that are required # for one of bundle creation. - ready, unready = ray.wait([pg.ready()], timeout=0) + ready, unready = ray.wait([pg.ready()], timeout=0.5) assert len(ready) == 0 assert len(unready) == 1 # Wait until all tasks are done. @@ -1233,17 +1240,13 @@ def test_create_actor_with_placement_group_after_gcs_server_restart( def test_create_placement_group_during_gcs_server_restart( ray_start_cluster_head): cluster = ray_start_cluster_head - cluster.add_node(num_cpus=20) + cluster.add_node(num_cpus=200) cluster.wait_for_nodes() # Create placement groups during gcs server restart. placement_groups = [] for i in range(0, 100): - placement_group = ray.util.placement_group([{ - "CPU": 0.1 - }, { - "CPU": 0.1 - }]) + placement_group = ray.util.placement_group([{"CPU": 1}, {"CPU": 1}]) placement_groups.append(placement_group) cluster.head_node.kill_gcs_server() @@ -1253,5 +1256,37 @@ def test_create_placement_group_during_gcs_server_restart( ray.get(placement_groups[i].ready()) +@pytest.mark.parametrize( + "ray_start_cluster_head", [ + generate_system_config_map( + num_heartbeats_timeout=20, ping_gcs_rpc_server_max_retries=60) + ], + indirect=True) +def test_placement_group_wait_api(ray_start_cluster_head): + cluster = ray_start_cluster_head + cluster.add_node(num_cpus=2) + cluster.add_node(num_cpus=2) + cluster.wait_for_nodes() + + # Create placement group 1 successfully. + placement_group1 = ray.util.placement_group([{"CPU": 1}, {"CPU": 1}]) + assert placement_group1.wait(10) + + # Restart gcs server. + cluster.head_node.kill_gcs_server() + cluster.head_node.start_gcs_server() + + # Create placement group 2 successfully. + placement_group2 = ray.util.placement_group([{"CPU": 1}, {"CPU": 1}]) + assert placement_group2.wait(10) + + # Remove placement group 1. + ray.util.remove_placement_group(placement_group1) + + # Wait for placement group 1 after it is removed. + with pytest.raises(Exception): + placement_group1.wait(10) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/tests/test_queue.py b/python/ray/tests/test_queue.py index a3f39ec4b..11b2a9a4c 100644 --- a/python/ray/tests/test_queue.py +++ b/python/ray/tests/test_queue.py @@ -5,6 +5,7 @@ from ray.exceptions import GetTimeoutError from ray.util.queue import Queue, Empty, Full +# Remote helper functions for testing concurrency @ray.remote def async_get(queue): return queue.get(block=True) @@ -50,6 +51,29 @@ def test_get(ray_start_regular_shared): q.get(timeout=0.2) +@pytest.mark.asyncio +async def test_get_async(ray_start_regular_shared): + + q = Queue() + + item = 0 + await q.put_async(item) + assert await q.get_async(block=False) == item + + item = 1 + await q.put_async(item) + assert await q.get_async(timeout=0.2) == item + + with pytest.raises(ValueError): + await q.get_async(timeout=-1) + + with pytest.raises(Empty): + await q.get_async(block=False) + + with pytest.raises(Empty): + await q.get_async(timeout=0.2) + + def test_put(ray_start_regular_shared): q = Queue(1) @@ -73,7 +97,31 @@ def test_put(ray_start_regular_shared): q.put(1, timeout=0.2) -def test_async_get(ray_start_regular_shared): +@pytest.mark.asyncio +async def test_put_async(ray_start_regular_shared): + + q = Queue(1) + + item = 0 + await q.put_async(item, block=False) + assert await q.get_async() == item + + item = 1 + await q.put_async(item, timeout=0.2) + assert await q.get_async() == item + + with pytest.raises(ValueError): + await q.put_async(0, timeout=-1) + + await q.put_async(0) + with pytest.raises(Full): + await q.put_async(1, block=False) + + with pytest.raises(Full): + await q.put_async(1, timeout=0.2) + + +def test_concurrent_get(ray_start_regular_shared): q = Queue() future = async_get.remote(q) @@ -87,7 +135,7 @@ def test_async_get(ray_start_regular_shared): assert ray.get(future) == 1 -def test_async_put(ray_start_regular_shared): +def test_concurrent_put(ray_start_regular_shared): q = Queue(1) q.put(1) future = async_put.remote(q, 2) @@ -102,6 +150,20 @@ def test_async_put(ray_start_regular_shared): assert q.get() == 2 +def test_batch(ray_start_regular_shared): + q = Queue(1) + + with pytest.raises(Full): + q.put_nowait_batch([1, 2]) + + with pytest.raises(Empty): + q.get_nowait_batch(1) + + big_q = Queue(100) + big_q.put_nowait_batch(list(range(100))) + assert big_q.get_nowait_batch(100) == list(range(100)) + + def test_qsize(ray_start_regular_shared): q = Queue() diff --git a/python/ray/tests/test_ray_debugger.py b/python/ray/tests/test_ray_debugger.py index 67df794f8..8007f8a7a 100644 --- a/python/ray/tests/test_ray_debugger.py +++ b/python/ray/tests/test_ray_debugger.py @@ -1,8 +1,11 @@ import json import os +import platform import sys from telnetlib import Telnet +import pexpect +import pytest import ray @@ -34,6 +37,71 @@ def test_ray_debugger_breakpoint(shutdown_only): ray.get(result) +@pytest.mark.skipif( + platform.system() == "Windows", reason="Failing on Windows.") +def test_ray_debugger_stepping(shutdown_only): + ray.init(num_cpus=1) + + @ray.remote + def g(): + return None + + @ray.remote + def f(): + ray.util.pdb.set_trace() + x = g.remote() + return ray.get(x) + + result = f.remote() + + p = pexpect.spawn("ray debug") + p.expect("Enter breakpoint index or press enter to refresh: ") + p.sendline("0") + p.expect("-> x = g.remote()") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("get") + p.expect("(Pdb)") + p.sendline("continue") + + # This should succeed now! + ray.get(result) + + +@pytest.mark.skipif( + platform.system() == "Windows", reason="Failing on Windows.") +def test_ray_debugger_recursive(shutdown_only): + ray.init(num_cpus=1) + + @ray.remote + def fact(n): + if n < 1: + return n + ray.util.pdb.set_trace() + n_id = fact.remote(n - 1) + return n * ray.get(n_id) + + result = fact.remote(5) + + p = pexpect.spawn("ray debug") + p.expect("Enter breakpoint index or press enter to refresh: ") + p.sendline("0") + p.expect("(Pdb)") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("remote") + p.expect("(Pdb)") + p.sendline("remote") + + ray.get(result) + + if __name__ == "__main__": import pytest # Make subprocess happy in bazel. diff --git a/python/ray/tests/test_reconstruction.py b/python/ray/tests/test_reconstruction.py index b471964c2..382225cea 100644 --- a/python/ray/tests/test_reconstruction.py +++ b/python/ray/tests/test_reconstruction.py @@ -9,7 +9,6 @@ import ray from ray.test_utils import ( wait_for_condition, wait_for_pid_to_exit, - new_scheduler_enabled, ) SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM @@ -488,7 +487,6 @@ def test_reconstruction_chain(ray_start_cluster, reconstruction_enabled): raise e.as_instanceof_cause() -@pytest.mark.skipif(new_scheduler_enabled(), reason="hangs") def test_reconstruction_stress(ray_start_cluster): config = { "num_heartbeats_timeout": 10, diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index f20462993..50d899af0 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -256,19 +256,19 @@ def test_add_min_workers_nodes(): } assert _add_min_workers_nodes([], {}, - types) == \ + types, None, None) == \ ([{"CPU": 2}]*50+[{"GPU": 1}]*99999, {"m2.large": 50, "gpu": 99999}, {"m2.large": 50, "gpu": 99999}) assert _add_min_workers_nodes([{"CPU": 2}]*5, {"m2.large": 5}, - types) == \ + types, None, None) == \ ([{"CPU": 2}]*50+[{"GPU": 1}]*99999, {"m2.large": 50, "gpu": 99999}, {"m2.large": 45, "gpu": 99999}) assert _add_min_workers_nodes([{"CPU": 2}]*60, {"m2.large": 60}, - types) == \ + types, None, None) == \ ([{"CPU": 2}]*60+[{"GPU": 1}]*99999, {"m2.large": 60, "gpu": 99999}, {"gpu": 99999}) @@ -279,7 +279,7 @@ def test_add_min_workers_nodes(): }] * 99999, { "m2.large": 50, "gpu": 99999 - }, types) == ([{ + }, types, None, None) == ([{ "CPU": 2 }] * 50 + [{ "GPU": 1 @@ -288,17 +288,18 @@ def test_add_min_workers_nodes(): "gpu": 99999 }, {}) - assert _add_min_workers_nodes([], {}, - {"gpubla": types["gpubla"]}) == ([], {}, {}) + assert _add_min_workers_nodes([], {}, {"gpubla": types["gpubla"]}, None, + None) == ([], {}, {}) types["gpubla"]["max_workers"] = 10 - assert _add_min_workers_nodes([], {}, {"gpubla": types["gpubla"]}) == ([{ - "GPU": 1 - }] * 10, { - "gpubla": 10 - }, { - "gpubla": 10 - }) + assert _add_min_workers_nodes([], {}, {"gpubla": types["gpubla"]}, None, + None) == ([{ + "GPU": 1 + }] * 10, { + "gpubla": 10 + }, { + "gpubla": 10 + }) def test_get_nodes_to_launch_with_min_workers(): @@ -1406,7 +1407,7 @@ class AutoscalingTest(unittest.TestCase): config_path = self.write_config(config) self.provider = MockProvider() runner = MockProcessRunner() - runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + runner.respond_to_call("json .Config.Env", ["[]" for i in range(3)]) autoscaler = StandardAutoscaler( config_path, LoadMetrics(), @@ -1432,6 +1433,7 @@ class AutoscalingTest(unittest.TestCase): sleep(0.1) runner.assert_has_call(self.provider.mock_nodes[1].internal_ip, "new_worker_setup_command") + runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip, "setup_cmd") runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip, @@ -1571,6 +1573,312 @@ class AutoscalingTest(unittest.TestCase): self.waitForNodes(2) assert self.provider.mock_nodes[1].node_type == "p2.8xlarge" + def testRequestResourcesIdleTimeout(self): + """Test request_resources() with and without idle timeout.""" + config = copy.deepcopy(MULTI_WORKER_CLUSTER) + config["max_workers"] = 4 + config["idle_timeout_minutes"] = 0 + config["available_node_types"] = { + "empty_node": { + "node_config": {}, + "resources": { + "CPU": 2 + }, + "max_workers": 1 + }, + "def_worker": { + "node_config": {}, + "resources": { + "CPU": 2, + "GPU": 1, + "WORKER": 1 + }, + "max_workers": 3 + } + } + config_path = self.write_config(config) + self.provider = MockProvider() + runner = MockProcessRunner() + lm = LoadMetrics() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + autoscaler = StandardAutoscaler( + config_path, + lm, + max_failures=0, + process_runner=runner, + update_interval_s=0) + autoscaler.update() + self.waitForNodes(0) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}]) + autoscaler.update() + self.waitForNodes(1) + non_terminated_nodes = autoscaler.provider.non_terminated_nodes({}) + assert len(non_terminated_nodes) == 1 + node_id = non_terminated_nodes[0] + node_ip = autoscaler.provider.non_terminated_node_ips({})[0] + + # A hack to check if the node was terminated when it shouldn't. + autoscaler.provider.mock_nodes[node_id].state = "unterminatable" + lm.update( + node_ip, + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }]) + autoscaler.update() + # this fits on request_resources()! + self.waitForNodes(1) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}] * 2) + autoscaler.update() + self.waitForNodes(2) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}]) + lm.update( + node_ip, + config["available_node_types"]["def_worker"]["resources"], {}, {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }]) + autoscaler.update() + self.waitForNodes(2) + lm.update( + node_ip, + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }]) + autoscaler.update() + # Still 2 as the second node did not show up a heart beat. + self.waitForNodes(2) + # If node {node_id} was terminated any time then it's state will be set + # to terminated. + assert autoscaler.provider.mock_nodes[ + node_id].state == "unterminatable" + lm.update( + "172.0.0.1", + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }]) + autoscaler.update() + # Now it is 1 because it showed up in last used (heart beat). + # The remaining one is 127.0.0.1. + self.waitForNodes(1) + + def testRequestResourcesRaceConditionsLong(self): + """Test request_resources(), race conditions & demands/min_workers. + + Tests when request_resources() is called simultaneously with resource + demands and min_workers constraint in multiple orders upscaling and + downscaling. + """ + config = copy.deepcopy(MULTI_WORKER_CLUSTER) + config["max_workers"] = 4 + config["idle_timeout_minutes"] = 0 + config["available_node_types"] = { + "empty_node": { + "node_config": {}, + "resources": { + "CPU": 2 + }, + "max_workers": 1 + }, + "def_worker": { + "node_config": {}, + "resources": { + "CPU": 2, + "GPU": 1, + "WORKER": 1 + }, + "max_workers": 3, + "min_workers": 1 + } + } + config_path = self.write_config(config) + self.provider = MockProvider() + runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(3)]) + lm = LoadMetrics() + autoscaler = StandardAutoscaler( + config_path, + lm, + max_failures=0, + process_runner=runner, + update_interval_s=0) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}]) + autoscaler.update() + # 1 min worker for both min_worker and request_resources() + self.waitForNodes(1) + non_terminated_nodes = autoscaler.provider.non_terminated_nodes({}) + assert len(non_terminated_nodes) == 1 + node_id = non_terminated_nodes[0] + node_ip = autoscaler.provider.non_terminated_node_ips({})[0] + + # A hack to check if the node was terminated when it shouldn't. + autoscaler.provider.mock_nodes[node_id].state = "unterminatable" + lm.update( + node_ip, + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }]) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}] * 2) + autoscaler.update() + # 2 requested_resource, 1 min worker, 1 free node -> 2 nodes total + self.waitForNodes(2) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}]) + autoscaler.update() + # Still 2 because the second one is not connected and hence + # request_resources occupies the connected node. + self.waitForNodes(2) + autoscaler.request_resources([{"CPU": 0.2, "WORKER": 1.0}] * 3) + lm.update( + node_ip, + config["available_node_types"]["def_worker"]["resources"], {}, {}, + waiting_bundles=[{ + "CPU": 0.2, + "WORKER": 1.0 + }] * 3) + autoscaler.update() + self.waitForNodes(3) + autoscaler.request_resources([]) + + lm.update("172.0.0.1", + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], + {}) + lm.update("172.0.0.2", + config["available_node_types"]["def_worker"]["resources"], + config["available_node_types"]["def_worker"]["resources"], + {}) + lm.update(node_ip, + config["available_node_types"]["def_worker"]["resources"], + {}, {}) + autoscaler.update() + self.waitForNodes(1) + # If node {node_id} was terminated any time then it's state will be set + # to terminated. + assert autoscaler.provider.mock_nodes[ + node_id].state == "unterminatable" + + def testRequestResourcesRaceConditionWithMinWorker(self): + """Test request_resources() with min_workers. + + Tests when request_resources() is called simultaneously with adding + min_workers constraint. + """ + config = copy.deepcopy(MULTI_WORKER_CLUSTER) + config["available_node_types"] = { + "empty_node": { + "node_config": {}, + "resources": { + "CPU": 2 + }, + "max_workers": 1 + }, + "def_worker": { + "node_config": {}, + "resources": { + "CPU": 2, + "GPU": 1, + "WORKER": 1 + }, + "max_workers": 3, + "min_workers": 1 + } + } + config_path = self.write_config(config) + self.provider = MockProvider() + runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + lm = LoadMetrics() + autoscaler = StandardAutoscaler( + config_path, + lm, + max_failures=0, + process_runner=runner, + update_interval_s=0) + autoscaler.request_resources([{"CPU": 2, "WORKER": 1.0}] * 2) + autoscaler.update() + # 2 min worker for both min_worker and request_resources(), not 3. + self.waitForNodes(2) + + def testRequestResourcesRaceConditionWithResourceDemands(self): + """Test request_resources() with resource_demands. + + Tests when request_resources() is called simultaneously with resource + demands in multiple orders. + """ + config = copy.deepcopy(MULTI_WORKER_CLUSTER) + config["available_node_types"].update({ + "empty_node": { + "node_config": {}, + "resources": { + "CPU": 2, + "GPU": 1 + }, + "max_workers": 1 + }, + "def_worker": { + "node_config": {}, + "resources": { + "CPU": 2, + "GPU": 1, + "WORKER": 1 + }, + "max_workers": 3, + } + }) + + config_path = self.write_config(config) + self.provider = MockProvider() + self.provider.create_node({}, { + TAG_RAY_NODE_KIND: "head", + TAG_RAY_USER_NODE_TYPE: "empty_node" + }, 1) + + runner = MockProcessRunner() + runner.respond_to_call("json .Config.Env", ["[]" for i in range(2)]) + lm = LoadMetrics() + autoscaler = StandardAutoscaler( + config_path, + lm, + max_failures=0, + process_runner=runner, + update_interval_s=0) + lm.update( + "127.0.0.0", { + "CPU": 2, + "GPU": 1 + }, {"CPU": 2}, {}, + waiting_bundles=[{ + "CPU": 2 + }]) + autoscaler.request_resources([{"CPU": 2, "GPU": 1}] * 2) + autoscaler.update() + # 1 head, 1 worker. + self.waitForNodes(2) + lm.update( + "127.0.0.0", { + "CPU": 2, + "GPU": 1 + }, {"CPU": 2}, {}, + waiting_bundles=[{ + "CPU": 2 + }]) + # make sure it stays consistent. + for _ in range(10): + autoscaler.update() + self.waitForNodes(2) + if __name__ == "__main__": import sys diff --git a/python/ray/tune/examples/dragonfly_example.py b/python/ray/tune/examples/dragonfly_example.py index 70f9f59e0..92f3c98b8 100644 --- a/python/ray/tune/examples/dragonfly_example.py +++ b/python/ray/tune/examples/dragonfly_example.py @@ -17,9 +17,9 @@ from ray.tune.suggest.dragonfly import DragonflySearch def objective(config): for i in range(config["iterations"]): - vol1 = config["point"][0] # LiNO3 - vol2 = config["point"][1] # Li2SO4 - vol3 = config["point"][2] # NaClO4 + vol1 = config["LiNO3_vol"] # LiNO3 + vol2 = config["Li2SO4_vol"] # Li2SO4 + vol3 = config["NaClO4_vol"] # NaClO4 vol4 = 10 - (vol1 + vol2 + vol3) # Water # Synthetic functions conductivity = vol1 + 0.1 * (vol2 + vol3)**2 + 2.3 * vol4 * (vol1**1.5) diff --git a/python/ray/tune/examples/hyperopt_example.py b/python/ray/tune/examples/hyperopt_example.py index 00afe29db..bb5131db4 100644 --- a/python/ray/tune/examples/hyperopt_example.py +++ b/python/ray/tune/examples/hyperopt_example.py @@ -40,12 +40,12 @@ if __name__ == "__main__": { "width": 1, "height": 2, - "activation": 0 # Activation will be relu + "activation": "relu" # Activation will be relu }, { "width": 4, "height": 2, - "activation": 1 # Activation will be tanh + "activation": "tanh" # Activation will be tanh } ] diff --git a/python/ray/tune/examples/skopt_example.py b/python/ray/tune/examples/skopt_example.py index 05274c535..a70c25c22 100644 --- a/python/ray/tune/examples/skopt_example.py +++ b/python/ray/tune/examples/skopt_example.py @@ -43,7 +43,18 @@ if __name__ == "__main__": # "activation": ["relu", "tanh"] # } - previously_run_params = [[10, 0, "relu"], [15, -20, "tanh"]] + previously_run_params = [ + { + "width": 10, + "height": 0, + "activation": "relu" # Activation will be relu + }, + { + "width": 15, + "height": -20, + "activation": "tanh" # Activation will be tanh + } + ] known_rewards = [-189, -1144] algo = SkOptSearch( diff --git a/python/ray/tune/experiment.py b/python/ray/tune/experiment.py index bf50951a3..4ad5c43c7 100644 --- a/python/ray/tune/experiment.py +++ b/python/ray/tune/experiment.py @@ -167,6 +167,13 @@ class Experiment: stopping_criteria = {} if not stop: pass + elif isinstance(stop, list): + if any(not isinstance(s, Stopper) for s in stop): + raise ValueError( + "If you pass a list as the `stop` argument to " + "`tune.run()`, each element must be an instance of " + "`tune.stopper.Stopper`.") + self._stopper = CombinedStopper(*stop) elif isinstance(stop, dict): stopping_criteria = stop elif callable(stop): diff --git a/python/ray/tune/integration/xgboost.py b/python/ray/tune/integration/xgboost.py index 730883c25..c3d6c02e9 100644 --- a/python/ray/tune/integration/xgboost.py +++ b/python/ray/tune/integration/xgboost.py @@ -1,14 +1,29 @@ from typing import Dict, List, Union +from collections import OrderedDict from ray import tune import os +from ray.tune.utils import flatten_dict +from xgboost.core import Booster -class TuneCallback: +try: + from xgboost.callback import TrainingCallback +except ImportError: + + class TrainingCallback: + pass + + +class TuneCallback(TrainingCallback): """Base class for Tune's XGBoost callbacks.""" - pass def __call__(self, env): + """Compatibility with xgboost<1.3""" + return self.after_iteration(env.model, env.iteration, + env.evaluation_result_list) + + def after_iteration(self, model: Booster, epoch: int, evals_log: Dict): raise NotImplementedError @@ -54,8 +69,15 @@ class TuneReportCallback(TuneCallback): metrics = [metrics] self._metrics = metrics - def __call__(self, env): - result_dict = dict(env.evaluation_result_list) + def _get_report_dict(self, evals_log): + if isinstance(evals_log, OrderedDict): + # xgboost>=1.3 + result_dict = flatten_dict(evals_log, delimiter="-") + for k in list(result_dict): + result_dict[k] = result_dict[k][-1] + else: + # xgboost<1.3 + result_dict = dict(evals_log) if not self._metrics: report_dict = result_dict else: @@ -66,6 +88,11 @@ class TuneReportCallback(TuneCallback): else: metric = key report_dict[key] = result_dict[metric] + return report_dict + + def after_iteration(self, model: Booster, epoch: int, evals_log: Dict): + + report_dict = self._get_report_dict(evals_log) tune.report(**report_dict) @@ -81,15 +108,25 @@ class _TuneCheckpointCallback(TuneCallback): Args: filename (str): Filename of the checkpoint within the checkpoint directory. Defaults to "checkpoint". + frequency (int): How often to save checkpoints. Per default, a + checkpoint is saved every five iterations. """ - def __init__(self, filename: str = "checkpoint"): + def __init__(self, filename: str = "checkpoint", frequency: int = 5): self._filename = filename + self._frequency = frequency - def __call__(self, env): - with tune.checkpoint_dir(step=env.iteration) as checkpoint_dir: - env.model.save_model(os.path.join(checkpoint_dir, self._filename)) + @staticmethod + def _create_checkpoint(model: Booster, epoch: int, filename: str, + frequency: int): + if epoch % frequency > 0: + return + with tune.checkpoint_dir(step=epoch) as checkpoint_dir: + model.save_model(os.path.join(checkpoint_dir, filename)) + + def after_iteration(self, model: Booster, epoch: int, evals_log: Dict): + self._create_checkpoint(model, epoch, self._filename, self._frequency) class TuneReportCheckpointCallback(TuneCallback): @@ -108,6 +145,8 @@ class TuneReportCheckpointCallback(TuneCallback): directory. Defaults to "checkpoint". If this is None, all metrics will be reported to Tune under their default names as obtained from XGBoost. + frequency (int): How often to save checkpoints. Per default, a + checkpoint is saved every five iterations. Example: @@ -132,13 +171,16 @@ class TuneReportCheckpointCallback(TuneCallback): {"loss": "eval-logloss"}, "xgboost.mdl)]) """ + _checkpoint_callback_cls = _TuneCheckpointCallback + _report_callbacks_cls = TuneReportCallback def __init__(self, metrics: Union[None, str, List[str], Dict[str, str]] = None, - filename: str = "checkpoint"): - self._checkpoint = _TuneCheckpointCallback(filename) - self._report = TuneReportCallback(metrics) + filename: str = "checkpoint", + frequency: int = 5): + self._checkpoint = self._checkpoint_callback_cls(filename, frequency) + self._report = self._report_callbacks_cls(metrics) - def __call__(self, env): - self._checkpoint(env) - self._report(env) + def after_iteration(self, model: Booster, epoch: int, evals_log: Dict): + self._checkpoint.after_iteration(model, epoch, evals_log) + self._report.after_iteration(model, epoch, evals_log) diff --git a/python/ray/tune/progress_reporter.py b/python/ray/tune/progress_reporter.py index 59c540dd8..a71a2da54 100644 --- a/python/ray/tune/progress_reporter.py +++ b/python/ray/tune/progress_reporter.py @@ -1,21 +1,26 @@ from __future__ import print_function -import collections -import sys +from typing import Dict, List, Optional, Union +import collections +import os +import sys import numpy as np import time +from ray.tune.callback import Callback +from ray.tune.logger import pretty_print from ray.tune.result import (DEFAULT_METRIC, EPISODE_REWARD_MEAN, MEAN_ACCURACY, MEAN_LOSS, TRAINING_ITERATION, TIME_TOTAL_S, TIMESTEPS_TOTAL, AUTO_RESULT_KEYS) -from ray.tune.trial import Trial +from ray.tune.trial import DEBUG_PRINT_INTERVAL, Trial from ray.tune.utils import unflattened_lookup +from ray.tune.utils.log import Verbosity, has_verbosity try: - from collections.abc import Mapping + from collections.abc import Mapping, MutableMapping except ImportError: - from collections import Mapping + from collections import Mapping, MutableMapping try: from tabulate import tabulate @@ -33,7 +38,7 @@ class ProgressReporter: receiving training results, and so on. """ - def should_report(self, trials, done=False): + def should_report(self, trials: List[Trial], done: bool = False): """Returns whether or not progress should be reported. Args: @@ -42,7 +47,7 @@ class ProgressReporter: """ raise NotImplementedError - def report(self, trials, done, *sys_info): + def report(self, trials: List[Trial], done: bool, *sys_info: Dict): """Reports progress across trials. Args: @@ -80,6 +85,12 @@ class TuneReporterBase(ProgressReporter): Defaults to 5s. infer_limit (int): Maximum number of metrics to automatically infer from tune results. + print_intermediate_tables (bool|None): Print intermediate result + tables. If None (default), will be set to True for verbosity + levels above 3, otherwise False. If True, intermediate tables + will be printed with experiment progress. If False, tables + will only be printed at then end of the tuning run for verbosity + levels greater than 2. metric (str): Metric used to determine best current trial. mode (str): One of [min, max]. Determines whether objective is minimizing or maximizing the metric attribute. @@ -99,16 +110,18 @@ class TuneReporterBase(ProgressReporter): type(None) } - def __init__(self, - metric_columns=None, - parameter_columns=None, - total_samples=None, - max_progress_rows=20, - max_error_rows=20, - max_report_frequency=5, - infer_limit=3, - metric=None, - mode=None): + def __init__( + self, + metric_columns: Union[None, List[str], Dict[str, str]] = None, + parameter_columns: Union[None, List[str], Dict[str, str]] = None, + total_samples: Optional[int] = None, + max_progress_rows: int = 20, + max_error_rows: int = 20, + max_report_frequency: int = 5, + infer_limit: int = 3, + print_intermediate_tables: Optional[bool] = None, + metric: Optional[str] = None, + mode: Optional[str] = None): self._total_samples = total_samples self._metrics_override = metric_columns is not None self._inferred_metrics = {} @@ -118,13 +131,20 @@ class TuneReporterBase(ProgressReporter): self._max_error_rows = max_error_rows self._infer_limit = infer_limit + if print_intermediate_tables is None: + self._print_intermediate_tables = has_verbosity( + Verbosity.V3_TRIAL_DETAILS) + else: + self._print_intermediate_tables = print_intermediate_tables + self._max_report_freqency = max_report_frequency self._last_report_time = 0 self._metric = metric self._mode = mode - def set_search_properties(self, metric, mode): + def set_search_properties(self, metric: Optional[str], + mode: Optional[str]): if self._metric and metric: return False if self._mode and mode: @@ -141,16 +161,18 @@ class TuneReporterBase(ProgressReporter): return True - def set_total_samples(self, total_samples): + def set_total_samples(self, total_samples: int): self._total_samples = total_samples - def should_report(self, trials, done=False): + def should_report(self, trials: List[Trial], done: bool = False): if time.time() - self._last_report_time > self._max_report_freqency: self._last_report_time = time.time() return True return done - def add_metric_column(self, metric, representation=None): + def add_metric_column(self, + metric: str, + representation: Optional[str] = None): """Adds a metric to the existing columns. Args: @@ -163,7 +185,7 @@ class TuneReporterBase(ProgressReporter): if metric in self._metric_columns: raise ValueError("Column {} already exists.".format(metric)) - if isinstance(self._metric_columns, Mapping): + if isinstance(self._metric_columns, MutableMapping): representation = representation or metric self._metric_columns[metric] = representation else: @@ -174,7 +196,9 @@ class TuneReporterBase(ProgressReporter): "of metric columns.") self._metric_columns.append(metric) - def add_parameter_column(self, parameter, representation=None): + def add_parameter_column(self, + parameter: str, + representation: Optional[str] = None): """Adds a parameter to the existing columns. Args: @@ -186,7 +210,7 @@ class TuneReporterBase(ProgressReporter): if parameter in self._parameter_columns: raise ValueError("Column {} already exists.".format(parameter)) - if isinstance(self._parameter_columns, Mapping): + if isinstance(self._parameter_columns, MutableMapping): representation = representation or parameter self._parameter_columns[parameter] = representation else: @@ -197,7 +221,12 @@ class TuneReporterBase(ProgressReporter): "of metric columns.") self._parameter_columns.append(parameter) - def _progress_str(self, trials, done, *sys_info, fmt="psql", delim="\n"): + def _progress_str(self, + trials: List[Trial], + done: bool, + *sys_info: Dict, + fmt: str = "psql", + delim: str = "\n"): """Returns full progress string. This string contains a progress table and error table. The progress @@ -228,19 +257,24 @@ class TuneReporterBase(ProgressReporter): best_trial_str(current_best_trial, metric, self._parameter_columns)) - messages.append( - trial_progress_str( - trials, - metric_columns=self._metric_columns, - parameter_columns=self._parameter_columns, - total_samples=self._total_samples, - fmt=fmt, - max_rows=max_progress)) - messages.append(trial_errors_str(trials, fmt=fmt, max_rows=max_error)) + if has_verbosity(Verbosity.V1_EXPERIMENT): + # Will filter the table in `trial_progress_str` + messages.append( + trial_progress_str( + trials, + metric_columns=self._metric_columns, + parameter_columns=self._parameter_columns, + total_samples=self._total_samples, + force_table=self._print_intermediate_tables, + fmt=fmt, + max_rows=max_progress, + done=done)) + messages.append( + trial_errors_str(trials, fmt=fmt, max_rows=max_error)) return delim.join(messages) + delim - def _infer_user_metrics(self, trials, limit=4): + def _infer_user_metrics(self, trials: List[Trial], limit: int = 4): """Try to infer the metrics to print out.""" if len(self._inferred_metrics) >= limit: return self._inferred_metrics @@ -258,7 +292,7 @@ class TuneReporterBase(ProgressReporter): return self._inferred_metrics return self._inferred_metrics - def _current_best_trial(self, trials): + def _current_best_trial(self, trials: List[Trial]): if not trials: return None, None @@ -309,26 +343,39 @@ class JupyterNotebookReporter(TuneReporterBase): corresponding to each trial. Defaults to 20. max_report_frequency (int): Maximum report frequency in seconds. Defaults to 5s. + infer_limit (int): Maximum number of metrics to automatically infer + from tune results. + print_intermediate_tables (bool|None): Print intermediate result + tables. If None (default), will be set to True for verbosity + levels above 3, otherwise False. If True, intermediate tables + will be printed with experiment progress. If False, tables + will only be printed at then end of the tuning run for verbosity + levels greater than 2. + metric (str): Metric used to determine best current trial. + mode (str): One of [min, max]. Determines whether objective is + minimizing or maximizing the metric attribute. """ - def __init__(self, - overwrite, - metric_columns=None, - parameter_columns=None, - total_samples=None, - max_progress_rows=20, - max_error_rows=20, - max_report_frequency=5, - infer_limit=3, - metric=None, - mode=None): - super(JupyterNotebookReporter, - self).__init__(metric_columns, parameter_columns, total_samples, - max_progress_rows, max_error_rows, - max_report_frequency, infer_limit, metric, mode) + def __init__( + self, + overwrite: bool, + metric_columns: Union[None, List[str], Dict[str, str]] = None, + parameter_columns: Union[None, List[str], Dict[str, str]] = None, + total_samples: Optional[int] = None, + max_progress_rows: int = 20, + max_error_rows: int = 20, + max_report_frequency: int = 5, + infer_limit: int = 3, + print_intermediate_tables: Optional[bool] = None, + metric: Optional[str] = None, + mode: Optional[str] = None): + super(JupyterNotebookReporter, self).__init__( + metric_columns, parameter_columns, total_samples, + max_progress_rows, max_error_rows, max_report_frequency, + infer_limit, print_intermediate_tables, metric, mode) self._overwrite = overwrite - def report(self, trials, done, *sys_info): + def report(self, trials: List[Trial], done: bool, *sys_info: Dict): from IPython.display import clear_output from IPython.core.display import display, HTML if self._overwrite: @@ -359,25 +406,38 @@ class CLIReporter(TuneReporterBase): corresponding to each trial. Defaults to 20. max_report_frequency (int): Maximum report frequency in seconds. Defaults to 5s. + infer_limit (int): Maximum number of metrics to automatically infer + from tune results. + print_intermediate_tables (bool|None): Print intermediate result + tables. If None (default), will be set to True for verbosity + levels above 3, otherwise False. If True, intermediate tables + will be printed with experiment progress. If False, tables + will only be printed at then end of the tuning run for verbosity + levels greater than 2. + metric (str): Metric used to determine best current trial. + mode (str): One of [min, max]. Determines whether objective is + minimizing or maximizing the metric attribute. """ - def __init__(self, - metric_columns=None, - parameter_columns=None, - total_samples=None, - max_progress_rows=20, - max_error_rows=20, - max_report_frequency=5, - infer_limit=3, - metric=None, - mode=None): + def __init__( + self, + metric_columns: Union[None, List[str], Dict[str, str]] = None, + parameter_columns: Union[None, List[str], Dict[str, str]] = None, + total_samples: Optional[int] = None, + max_progress_rows: int = 20, + max_error_rows: int = 20, + max_report_frequency: int = 5, + infer_limit: int = 3, + print_intermediate_tables: Optional[bool] = None, + metric: Optional[str] = None, + mode: Optional[str] = None): - super(CLIReporter, - self).__init__(metric_columns, parameter_columns, total_samples, - max_progress_rows, max_error_rows, - max_report_frequency, infer_limit, metric, mode) + super(CLIReporter, self).__init__( + metric_columns, parameter_columns, total_samples, + max_progress_rows, max_error_rows, max_report_frequency, + infer_limit, print_intermediate_tables, metric, mode) - def report(self, trials, done, *sys_info): + def report(self, trials: List[Trial], done: bool, *sys_info: Dict): print(self._progress_str(trials, done, *sys_info)) @@ -400,15 +460,25 @@ def memory_debug_str(): round(used_gb, 1), round(total_gb, 1), warn) except ImportError: return ("Unknown memory usage. Please run `pip install psutil` " - "(or ray[debug]) to resolve)") + "to resolve)") -def trial_progress_str(trials, - metric_columns, - parameter_columns=None, - total_samples=0, - fmt="psql", - max_rows=None): +def _get_trials_by_state(trials: List[Trial]): + trials_by_state = collections.defaultdict(list) + for t in trials: + trials_by_state[t.status].append(t) + return trials_by_state + + +def trial_progress_str( + trials: List[Trial], + metric_columns: Union[List[str], Dict[str, str]], + parameter_columns: Union[None, List[str], Dict[str, str]] = None, + total_samples: int = 0, + force_table: bool = False, + fmt: str = "psql", + max_rows: Optional[int] = None, + done: bool = False): """Returns a human readable message for printing to the console. This contains a table where each row represents a trial, its parameters @@ -426,9 +496,13 @@ def trial_progress_str(trials, the parameter name is used in the message directly. If this is empty, all parameters are used in the message. total_samples (int): Total number of trials that will be generated. + force_table (bool): Force printing a table. If False, a table will + be printed only at the end of the training for verbosity levels + above `Verbosity.V2_TRIAL_NORM`. fmt (str): Output format (see tablefmt in tabulate API). max_rows (int): Maximum number of rows in the trial table. Defaults to unlimited. + done (bool): True indicates that the tuning run finished. """ messages = [] delim = "
" if fmt == "html" else "\n" @@ -436,9 +510,7 @@ def trial_progress_str(trials, return delim.join(messages) num_trials = len(trials) - trials_by_state = collections.defaultdict(list) - for t in trials: - trials_by_state[t.status].append(t) + trials_by_state = _get_trials_by_state(trials) for local_dir in sorted({t.local_dir for t in trials}): messages.append("Result logdir: {}".format(local_dir)) @@ -448,6 +520,30 @@ def trial_progress_str(trials, for state in sorted(trials_by_state) ] + if total_samples and total_samples >= sys.maxsize: + total_samples = "infinite" + + messages.append("Number of trials: {}{} ({})".format( + num_trials, f"/{total_samples}" + if total_samples else "", ", ".join(num_trials_strs))) + + if force_table or (has_verbosity(Verbosity.V2_TRIAL_NORM) and done): + messages += trial_progress_table(trials, metric_columns, + parameter_columns, fmt, max_rows) + + return delim.join(messages) + + +def trial_progress_table( + trials: List[Trial], + metric_columns: Union[List[str], Dict[str, str]], + parameter_columns: Union[None, List[str], Dict[str, str]] = None, + fmt: str = "psql", + max_rows: Optional[int] = None): + messages = [] + num_trials = len(trials) + trials_by_state = _get_trials_by_state(trials) + state_tbl_order = [ Trial.RUNNING, Trial.PAUSED, Trial.PENDING, Trial.TERMINATED, Trial.ERROR @@ -472,19 +568,13 @@ def trial_progress_str(trials, overflow_str = ", ".join(overflow_strs) else: overflow = False + overflow_str = "" trials = [] for state in state_tbl_order: if state not in trials_by_state: continue trials += trials_by_state[state] - if total_samples and total_samples >= sys.maxsize: - total_samples = "infinite" - - messages.append("Number of trials: {}{} ({})".format( - num_trials, f"/{total_samples}" - if total_samples else "", ", ".join(num_trials_strs))) - # Pre-process trials to figure out what columns to show. if isinstance(metric_columns, Mapping): metric_keys = list(metric_columns.keys()) @@ -526,10 +616,12 @@ def trial_progress_str(trials, if overflow: messages.append("... {} more trials not shown ({})".format( overflow, overflow_str)) - return delim.join(messages) + return messages -def trial_errors_str(trials, fmt="psql", max_rows=None): +def trial_errors_str(trials: List[Trial], + fmt: str = "psql", + max_rows: Optional[int] = None): """Returns a readable message regarding trial errors. Args: @@ -558,7 +650,10 @@ def trial_errors_str(trials, fmt="psql", max_rows=None): return delim.join(messages) -def best_trial_str(trial, metric, parameter_columns=None): +def best_trial_str( + trial: Trial, + metric: str, + parameter_columns: Union[None, List[str], Dict[str, str]] = None): """Returns a readable message stating the current best trial.""" val = trial.last_result[metric] config = trial.last_result.get("config", {}) @@ -570,7 +665,8 @@ def best_trial_str(trial, metric, parameter_columns=None): f"parameters={params}" -def _fair_filter_trials(trials_by_state, max_trials): +def _fair_filter_trials(trials_by_state: Dict[str, List[Trial]], + max_trials: int): """Filters trials such that each state is represented fairly. The oldest trials are truncated if necessary. @@ -605,7 +701,7 @@ def _fair_filter_trials(trials_by_state, max_trials): return filtered_trials -def _get_trial_info(trial, parameters, metrics): +def _get_trial_info(trial: Trial, parameters: List[str], metrics: List[str]): """Returns the following information about a trial: name | status | loc | params... | metrics... @@ -625,3 +721,109 @@ def _get_trial_info(trial, parameters, metrics): unflattened_lookup(metric, result, default=None) for metric in metrics ] return trial_info + + +class TrialProgressCallback(Callback): + """Reports (prints) intermediate trial progress. + + This callback is automatically added to the callback stack. When a + result is obtained, this callback will print the results according to + the specified verbosity level. + + For ``Verbosity.V3_TRIAL_DETAILS``, a full result list is printed. + + For ``Verbosity.V2_TRIAL_NORM``, only one line is printed per received + result. + + All other verbosity levels do not print intermediate trial progress. + + Result printing is throttled on a per-trial basis. Per default, results are + printed only once every 30 seconds. Results are always printed when a trial + finished or errored. + + """ + + def __init__(self, metric: Optional[str] = None): + self._last_print = collections.defaultdict(float) + self._completed_trials = set() + self._last_result_str = {} + self._metric = metric + + def on_trial_result(self, iteration: int, trials: List["Trial"], + trial: "Trial", result: Dict, **info): + self.log_result(trial, result, error=False) + + def on_trial_error(self, iteration: int, trials: List["Trial"], + trial: "Trial", **info): + self.log_result(trial, trial.last_result, error=True) + + def on_trial_complete(self, iteration: int, trials: List["Trial"], + trial: "Trial", **info): + # Only log when we never logged that a trial was completed + if trial not in self._completed_trials: + self._completed_trials.add(trial) + + print_result_str = self._print_result(trial.last_result) + last_result_str = self._last_result_str.get(trial, "") + # If this is a new result, print full result string + if print_result_str != last_result_str: + self.log_result(trial, trial.last_result, error=False) + else: + print(f"Trial {trial} completed. " + f"Last result: {print_result_str}") + + def log_result(self, trial: "Trial", result: Dict, error: bool = False): + done = result.get("done", False) is True + last_print = self._last_print[trial] + if done and trial not in self._completed_trials: + self._completed_trials.add(trial) + if has_verbosity(Verbosity.V3_TRIAL_DETAILS) and \ + (done or error or time.time() - last_print > DEBUG_PRINT_INTERVAL): + print("Result for {}:".format(trial)) + print(" {}".format(pretty_print(result).replace("\n", "\n "))) + self._last_print[trial] = time.time() + elif has_verbosity(Verbosity.V2_TRIAL_NORM) and ( + done or error + or time.time() - last_print > DEBUG_PRINT_INTERVAL): + info = "" + if done: + info = " This trial completed." + + metric_name = self._metric or "_metric" + metric_value = result.get(metric_name, -99.) + + print_result_str = self._print_result(result) + + self._last_result_str[trial] = print_result_str + + error_file = os.path.join(trial.logdir, "error.txt") + + if error: + message = f"The trial {trial} errored with " \ + f"parameters={trial.config}. " \ + f"Error file: {error_file}" + elif self._metric: + message = f"Trial {trial} reported " \ + f"{metric_name}={metric_value:.2f} " \ + f"with parameters={trial.config}.{info}" + else: + message = f"Trial {trial} reported " \ + f"{print_result_str} " \ + f"with parameters={trial.config}.{info}" + + print(message) + self._last_print[trial] = time.time() + + def _print_result(self, result: Dict): + print_result = result.copy() + print_result.pop("config", None) + print_result.pop("hist_stats", None) + print_result.pop("trial_id", None) + print_result.pop("experiment_tag", None) + print_result.pop("done", None) + for auto_result in AUTO_RESULT_KEYS: + print_result.pop(auto_result, None) + + print_result_str = ",".join( + [f"{k}={v}" for k, v in print_result.items()]) + return print_result_str diff --git a/python/ray/tune/registry.py b/python/ray/tune/registry.py index 5ffa4e416..250e8b8b2 100644 --- a/python/ray/tune/registry.py +++ b/python/ray/tune/registry.py @@ -79,11 +79,11 @@ def register_env(name, env_creator): Args: name (str): Name to register. - env_creator (obj): Function that creates an env. + env_creator (obj): Callable that creates an env. """ - if not isinstance(env_creator, FunctionType): - raise TypeError("Second argument must be a function.", env_creator) + if not callable(env_creator): + raise TypeError("Second argument must be callable.", env_creator) _global_registry.register(ENV_CREATOR, name, env_creator) diff --git a/python/ray/tune/stopper.py b/python/ray/tune/stopper.py index 9dc20773d..bc0940bd7 100644 --- a/python/ray/tune/stopper.py +++ b/python/ray/tune/stopper.py @@ -1,4 +1,7 @@ +import warnings +from typing import Dict, Optional import time +from collections import defaultdict, deque import numpy as np @@ -43,6 +46,27 @@ class Stopper: class CombinedStopper(Stopper): + """Combine several stoppers via 'OR'. + + Args: + *stoppers (Stopper): Stoppers to be combined. + + Example: + + .. code-block:: python + + from ray.tune.stopper import CombinedStopper, \ + MaximumIterationStopper, TrialPlateauStopper + + stopper = CombinedStopper( + MaximumIterationStopper(max_iter=20), + TrialPlateauStopper(metric="my_metric") + ) + + tune.run(train, stop=stopper) + + """ + def __init__(self, *stoppers: Stopper): self._stoppers = stoppers @@ -62,6 +86,18 @@ class NoopStopper(Stopper): class FunctionStopper(Stopper): + """Provide a custom function to check if trial should be stopped. + + The passed function will be called after each iteration. If it returns + True, the trial will be stopped. + + Args: + function (Callable[[str, Dict], bool): Function that checks if a trial + should be stopped. Must accept the `trial_id` string and `result` + dictionary as arguments. Must return a boolean. + + """ + def __init__(self, function): self._fn = function @@ -81,33 +117,53 @@ class FunctionStopper(Stopper): return is_function -class EarlyStopping(Stopper): +class MaximumIterationStopper(Stopper): + """Stop trials after reaching a maximum number of iterations + + Args: + max_iter (int): Number of iterations before stopping a trial. + """ + + def __init__(self, max_iter: int): + self._max_iter = max_iter + self._iter = defaultdict(lambda: 0) + + def __call__(self, trial_id: str, result: Dict): + self._iter[trial_id] += 1 + return self._iter[trial_id] >= self._max_iter + + def stop_all(self): + return False + + +class ExperimentPlateauStopper(Stopper): + """Early stop the experiment when a metric plateaued across trials. + + Stops the entire experiment when the metric has plateaued + for more than the given amount of iterations specified in + the patience parameter. + + Args: + metric (str): The metric to be monitored. + std (float): The minimal standard deviation after which + the tuning process has to stop. + top (int): The number of best models to consider. + mode (str): The mode to select the top results. + Can either be "min" or "max". + patience (int): Number of epochs to wait for + a change in the top models. + + Raises: + ValueError: If the mode parameter is not "min" nor "max". + ValueError: If the top parameter is not an integer + greater than 1. + ValueError: If the standard deviation parameter is not + a strictly positive float. + ValueError: If the patience parameter is not + a strictly positive integer. + """ + def __init__(self, metric, std=0.001, top=10, mode="min", patience=0): - """Create the EarlyStopping object. - - Stops the entire experiment when the metric has plateaued - for more than the given amount of iterations specified in - the patience parameter. - - Args: - metric (str): The metric to be monitored. - std (float): The minimal standard deviation after which - the tuning process has to stop. - top (int): The number of best model to consider. - mode (str): The mode to select the top results. - Can either be "min" or "max". - patience (int): Number of epochs to wait for - a change in the top models. - - Raises: - ValueError: If the mode parameter is not "min" nor "max". - ValueError: If the top parameter is not an integer - greater than 1. - ValueError: If the standard deviation parameter is not - a strictly positive float. - ValueError: If the patience parameter is not - a strictly positive integer. - """ if mode not in ("min", "max"): raise ValueError("The mode parameter can only be" " either min or max.") @@ -157,9 +213,107 @@ class EarlyStopping(Stopper): return self.has_plateaued() and self._iterations >= self._patience +class EarlyStopping(ExperimentPlateauStopper): + def __init__(self, *args, **kwargs): + warnings.warn( + "The `EarlyStopping` stopper has been renamed to " + "`ExperimentPlateauStopper`. The reference will be removed " + "in a future version of Ray. Please use ExperimentPlateauStopper" + "instead.", DeprecationWarning) + super(EarlyStopping, self).__init__(*args, **kwargs) + + +class TrialPlateauStopper(Stopper): + """Early stop single trials when they reached a plateau. + + When the standard deviation of the `metric` result of a trial is + below a threshold `std`, the trial plateaued and will be stopped + early. + + Args: + metric (str): Metric to check for convergence. + std (float): Maximum metric standard deviation to decide if a + trial plateaued. Defaults to 0.01. + num_results (int): Number of results to consider for stdev + calculation. + grace_period (int): Minimum number of timesteps before a trial + can be early stopped + metric_threshold (Optional[float]): + Minimum or maximum value the result has to exceed before it can + be stopped early. + mode (Optional[str]): If a `metric_threshold` argument has been + passed, this must be one of [min, max]. Specifies if we optimize + for a large metric (max) or a small metric (min). If max, the + `metric_threshold` has to be exceeded, if min the value has to + be lower than `metric_threshold` in order to early stop. + """ + + def __init__(self, + metric: str, + std: float = 0.01, + num_results: int = 4, + grace_period: int = 4, + metric_threshold: Optional[float] = None, + mode: Optional[str] = None): + self._metric = metric + self._mode = mode + + self._std = std + self._num_results = num_results + self._grace_period = grace_period + self._metric_threshold = metric_threshold + + if self._metric_threshold: + if mode not in ["min", "max"]: + raise ValueError( + f"When specifying a `metric_threshold`, the `mode` " + f"argument has to be one of [min, max]. " + f"Got: {mode}") + + self._iter = defaultdict(lambda: 0) + self._trial_results = defaultdict( + lambda: deque(maxlen=self._num_results)) + + def __call__(self, trial_id: str, result: Dict): + metric_result = result.get(self._metric) + self._trial_results[trial_id].append(metric_result) + self._iter[trial_id] += 1 + + # If still in grace period, do not stop yet + if self._iter[trial_id] < self._grace_period: + return False + + # If not enough results yet, do not stop yet + if len(self._trial_results[trial_id]) < self._num_results: + return False + + # If metric threshold value not reached, do not stop yet + if self._metric_threshold is not None: + if self._mode == "min" and metric_result > self._metric_threshold: + return False + elif self._mode == "max" and \ + metric_result < self._metric_threshold: + return False + + # Calculate stdev of last `num_results` results + try: + current_std = np.std(self._trial_results[trial_id]) + except Exception: + current_std = float("inf") + + # If stdev is lower than threshold, stop early. + return current_std < self._std + + def stop_all(self): + return False + + class TimeoutStopper(Stopper): """Stops all trials after a certain timeout. + This stopper is automatically created when the `time_budget_s` + argument is passed to `tune.run()`. + Args: timeout (int|float|datetime.timedelta): Either a number specifying the timeout in seconds, or a `datetime.timedelta` object. diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py index e2540fe0a..5f15611bf 100644 --- a/python/ray/tune/suggest/ax.py +++ b/python/ray/tune/suggest/ax.py @@ -1,3 +1,4 @@ +import copy from typing import Dict, List, Optional, Union from ax.service.ax_client import AxClient @@ -50,6 +51,11 @@ class AxSearch(Searcher): the `ray.tune.result.DEFAULT_METRIC` will be used per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. Defaults to "max". + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. parameter_constraints (list[str]): Parameter constraints, such as "x3 >= x4" or "x3 + x4 >= 2". outcome_constraints (list[str]): Outcome constraints of form @@ -110,6 +116,7 @@ class AxSearch(Searcher): space: Optional[Union[Dict, List[Dict]]] = None, metric: Optional[str] = None, mode: Optional[str] = None, + points_to_evaluate: Optional[List[Dict]] = None, parameter_constraints: Optional[List] = None, outcome_constraints: Optional[List] = None, ax_client: Optional[AxClient] = None, @@ -141,6 +148,8 @@ class AxSearch(Searcher): self._parameter_constraints = parameter_constraints self._outcome_constraints = outcome_constraints + self._points_to_evaluate = copy.deepcopy(points_to_evaluate) + self.max_concurrent = max_concurrent self._objective_name = metric @@ -226,7 +235,13 @@ class AxSearch(Searcher): if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None - parameters, trial_index = self._ax.get_next_trial() + + if self._points_to_evaluate: + config = self._points_to_evaluate.pop(0) + parameters, trial_index = self._ax.attach_trial(config) + else: + parameters, trial_index = self._ax.get_next_trial() + self._live_trial_mapping[trial_id] = trial_index return unflatten_dict(parameters) diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py index a3620bb17..df8f94546 100644 --- a/python/ray/tune/suggest/bayesopt.py +++ b/python/ray/tune/suggest/bayesopt.py @@ -2,7 +2,7 @@ from collections import defaultdict import logging import pickle import json -from typing import Dict, Optional, Tuple +from typing import Dict, List, Optional, Tuple from ray.tune import ExperimentAnalysis from ray.tune.result import DEFAULT_METRIC @@ -59,6 +59,11 @@ class BayesOptSearch(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. utility_kwargs (dict): Parameters to define the utility function. The default value is a dictionary with three keys: - kind: ucb (Upper Confidence Bound) @@ -112,6 +117,7 @@ class BayesOptSearch(Searcher): space: Optional[Dict] = None, metric: Optional[str] = None, mode: Optional[str] = None, + points_to_evaluate: Optional[List[Dict]] = None, utility_kwargs: Optional[Dict] = None, random_state: int = 42, random_search_steps: int = 10, @@ -121,35 +127,6 @@ class BayesOptSearch(Searcher): analysis: Optional[ExperimentAnalysis] = None, max_concurrent: Optional[int] = None, use_early_stopped_trials: Optional[bool] = None): - """Instantiate new BayesOptSearch object. - - Args: - space (dict): Continuous search space. - Parameters will be sampled from - this space which will be used to run trials. - metric (str): The training result objective value attribute. - mode (str): One of {min, max}. Determines whether objective is - minimizing or maximizing the metric attribute. - utility_kwargs (dict): Parameters to define the utility function. - Must provide values for the keys `kind`, `kappa`, and `xi`. - random_state (int): Used to initialize BayesOpt. - random_search_steps (int): Number of initial random searches. - This is necessary to avoid initial local overfitting - of the Bayesian process. - patience (int): Must be > 0. If the optimizer suggests a set of - hyperparameters more than 'patience' times, - then the whole experiment will stop. - skip_duplicate (bool): If true, BayesOptSearch will not create - a trial with a previously seen set of hyperparameters. By - default, floating values will be reduced to a digit precision - of 5. You can override this by setting - ``searcher.repeat_float_precision``. - analysis (ExperimentAnalysis): Optionally, the previous analysis - to integrate. - verbose (int): Sets verbosity level for BayesOpt packages. - max_concurrent: Deprecated. - use_early_stopped_trials: Deprecated. - """ assert byo is not None, ( "BayesOpt must be installed!. You can install BayesOpt with" " the command: `pip install bayesian-optimization`.") @@ -183,6 +160,8 @@ class BayesOptSearch(Searcher): elif mode == "min": self._metric_op = -1. + self._points_to_evaluate = points_to_evaluate + self._live_trial_mapping = {} self._buffered_trial_results = [] self.random_search_trials = random_search_steps @@ -269,8 +248,11 @@ class BayesOptSearch(Searcher): # we stop the suggestion and return None. return None - # We compute the new point to explore - config = self.optimizer.suggest(self.utility) + if self._points_to_evaluate: + config = self._points_to_evaluate.pop(0) + else: + # We compute the new point to explore + config = self.optimizer.suggest(self.utility) config_hash = _dict_hash(config, self.repeat_float_precision) # Check if already computed @@ -369,16 +351,16 @@ class BayesOptSearch(Searcher): def save(self, checkpoint_path: str): """Storing current optimizer state.""" with open(checkpoint_path, "wb") as f: - pickle.dump( - (self.optimizer, self._buffered_trial_results, - self._total_random_search_trials, self._config_counter), f) + pickle.dump((self.optimizer, self._buffered_trial_results, + self._total_random_search_trials, + self._config_counter, self._points_to_evaluate), f) def restore(self, checkpoint_path: str): """Restoring current optimizer state.""" with open(checkpoint_path, "rb") as f: (self.optimizer, self._buffered_trial_results, - self._total_random_search_trials, - self._config_counter) = pickle.load(f) + self._total_random_search_trials, self._config_counter, + self._points_to_evaluate) = pickle.load(f) @staticmethod def convert_search_space(spec: Dict, join: bool = False) -> Dict: @@ -403,7 +385,7 @@ class BayesOptSearch(Searcher): logger.warning( "BayesOpt does not support specific sampling methods. " "The {} sampler will be dropped.".format(sampler)) - return (domain.lower, domain.upper) + return (domain.lower, domain.upper) raise ValueError("BayesOpt does not support parameters of type " "`{}`".format(type(domain).__name__)) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index e6e803d30..b173d5e85 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -3,7 +3,7 @@ import copy import logging import math -from typing import Dict, Optional, Union +from typing import Dict, List, Optional, Union import ConfigSpace from ray.tune.result import DEFAULT_METRIC @@ -51,6 +51,11 @@ class TuneBOHB(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. seed (int): Optional random seed to initialize the random number generator. Setting this should lead to identical initial configurations at each run. @@ -107,6 +112,7 @@ class TuneBOHB(Searcher): max_concurrent: int = 10, metric: Optional[str] = None, mode: Optional[str] = None, + points_to_evaluate: Optional[List[Dict]] = None, seed: Optional[int] = None): from hpbandster.optimizers.config_generators.bohb import BOHB assert BOHB is not None, """HpBandSter must be installed! @@ -133,6 +139,8 @@ class TuneBOHB(Searcher): self._space = space self._seed = seed + self._points_to_evaluate = points_to_evaluate + super(TuneBOHB, self).__init__(metric=self._metric, mode=mode) if self._space: @@ -185,8 +193,11 @@ class TuneBOHB(Searcher): mode=self._mode)) if len(self.running) < self._max_concurrent: - # This parameter is not used in hpbandster implementation. - config, info = self.bohber.get_config(None) + if self._points_to_evaluate: + config = self._points_to_evaluate.pop(0) + else: + # This parameter is not used in hpbandster implementation. + config, info = self.bohber.get_config(None) self.trial_to_params[trial_id] = copy.deepcopy(config) self.running.add(trial_id) return unflatten_dict(config) diff --git a/python/ray/tune/suggest/dragonfly.py b/python/ray/tune/suggest/dragonfly.py index 4be1f47dd..3037fbbbf 100644 --- a/python/ray/tune/suggest/dragonfly.py +++ b/python/ray/tune/suggest/dragonfly.py @@ -12,7 +12,7 @@ from ray.tune.sample import Domain, Float, Quantized from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars -from ray.tune.utils.util import flatten_dict, is_nan_or_inf +from ray.tune.utils.util import flatten_dict, is_nan_or_inf, unflatten_dict try: # Python 3 only -- needed for lint test. import dragonfly @@ -68,11 +68,11 @@ class DragonflySearch(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. - points_to_evaluate (list of lists): A list of points you'd like to run - first before sampling from the optimiser, e.g. these could be - parameter configurations you already know work well to help - the optimiser select good values. Each point is a list of the - parameters using the order definition given by parameter_names. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. evaluated_rewards (list): If you have previously evaluated the parameters passed in as points_to_evaluate you can avoid re-running those trials by passing in the reward attributes @@ -142,7 +142,7 @@ class DragonflySearch(Searcher): space: Optional[Union[Dict, List[Dict]]] = None, metric: Optional[str] = None, mode: Optional[str] = None, - points_to_evaluate: Optional[List[List]] = None, + points_to_evaluate: Optional[List[Dict]] = None, evaluated_rewards: Optional[List] = None, **kwargs): assert dragonfly is not None, """dragonfly must be installed! @@ -170,6 +170,7 @@ class DragonflySearch(Searcher): self._evaluated_rewards = evaluated_rewards self._initial_points = [] self._live_trial_mapping = {} + self._point_parameter_names = [] self._opt = None if isinstance(optimizer, BlackboxOptimiser): @@ -206,6 +207,8 @@ class DragonflySearch(Searcher): "You have to set a `domain` when initializing dragonfly. " "Choose one of [Cartesian, Euclidean].") + self._point_parameter_names = [param["name"] for param in self._space] + if self._domain.lower().startswith("cartesian"): function_caller_cls = CPFunctionCaller elif self._domain.lower().startswith("euclidean"): @@ -250,12 +253,18 @@ class DragonflySearch(Searcher): self.init_dragonfly() def init_dragonfly(self): + if self._points_to_evaluate: + points_to_evaluate = [[ + config[par] for par in self._point_parameter_names + ] for config in self._points_to_evaluate] + else: + points_to_evaluate = None + self._opt.initialise() - if self._points_to_evaluate and self._evaluated_rewards: - self._opt.tell([(self._points_to_evaluate, - self._evaluated_rewards)]) - elif self._points_to_evaluate: - self._initial_points = self._points_to_evaluate + if points_to_evaluate and self._evaluated_rewards: + self._opt.tell([(points_to_evaluate, self._evaluated_rewards)]) + elif points_to_evaluate: + self._initial_points = points_to_evaluate # Dragonfly internally maximizes, so "min" => -1 if self._mode == "min": self._metric_op = -1. @@ -306,7 +315,11 @@ class DragonflySearch(Searcher): "parallelism in the experiment: %s", str(exc)) return None self._live_trial_mapping[trial_id] = suggested_config - return {"point": suggested_config} + + config = dict(zip(self._point_parameter_names, suggested_config)) + # Keep backwards compatibility + config.update(point=suggested_config) + return unflatten_dict(config) def on_trial_complete(self, trial_id: str, diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py index 3f0b1a939..aee5fd82d 100644 --- a/python/ray/tune/suggest/hyperopt.py +++ b/python/ray/tune/suggest/hyperopt.py @@ -58,11 +58,9 @@ class HyperOptSearch(Searcher): minimizing or maximizing the metric attribute. points_to_evaluate (list): Initial parameter suggestions to be run first. This is for when you already have some good parameters - you want hyperopt to run first to help the TPE algorithm - make better suggestions for future parameters. Needs to be - a list of dict of hyperopt-named variables. - Choice variables should be indicated by their index in the - list (see example) + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. n_initial_points (int): number of random evaluations of the objective function before starting to aproximate it with tree parzen estimators. Defaults to 20. @@ -86,7 +84,7 @@ class HyperOptSearch(Searcher): current_best_params = [{ 'width': 10, 'height': 0, - 'activation': 0, # The index of "relu" + 'activation': "relu", }] hyperopt_search = HyperOptSearch( @@ -109,7 +107,7 @@ class HyperOptSearch(Searcher): current_best_params = [{ 'width': 10, 'height': 0, - 'activation': 0, # The index of "relu" + 'activation': "relu", }] hyperopt_search = HyperOptSearch( @@ -137,7 +135,6 @@ class HyperOptSearch(Searcher): "HyperOpt must be installed! Run `pip install hyperopt`.") if mode: assert mode in ["min", "max"], "`mode` must be 'min' or 'max'." - from hyperopt.fmin import generate_trials_to_calculate super(HyperOptSearch, self).__init__( metric=metric, mode=mode, @@ -157,15 +154,9 @@ class HyperOptSearch(Searcher): hpo.tpe.suggest, n_startup_jobs=n_initial_points) if gamma is not None: self.algo = partial(self.algo, gamma=gamma) - if points_to_evaluate is None: - self._hpopt_trials = hpo.Trials() - self._points_to_evaluate = 0 - else: - assert isinstance(points_to_evaluate, (list, tuple)) - self._hpopt_trials = generate_trials_to_calculate( - points_to_evaluate) - self._hpopt_trials.refresh() - self._points_to_evaluate = len(points_to_evaluate) + + self._points_to_evaluate = copy.deepcopy(points_to_evaluate) + self._live_trial_mapping = {} if random_state_seed is None: self.rstate = np.random.RandomState() @@ -184,12 +175,69 @@ class HyperOptSearch(Searcher): self._setup_hyperopt() def _setup_hyperopt(self): + from hyperopt.fmin import generate_trials_to_calculate + if self._metric is None and self._mode: # If only a mode was passed, use anonymous metric self._metric = DEFAULT_METRIC + if self._points_to_evaluate is None: + self._hpopt_trials = hpo.Trials() + self._points_to_evaluate = 0 + else: + assert isinstance(self._points_to_evaluate, (list, tuple)) + + for i in range(len(self._points_to_evaluate)): + config = self._points_to_evaluate[i] + self._convert_categories_to_indices(config) + # HyperOpt treats initial points as LIFO, reverse to get FIFO + self._points_to_evaluate = list(reversed(self._points_to_evaluate)) + self._hpopt_trials = generate_trials_to_calculate( + self._points_to_evaluate) + self._hpopt_trials.refresh() + self._points_to_evaluate = len(self._points_to_evaluate) + self.domain = hpo.Domain(lambda spc: spc, self._space) + def _convert_categories_to_indices(self, config): + """Convert config parameters for categories into hyperopt-compatible + representations where instead the index of the category is expected.""" + + def _lookup(config_dict, space_dict, key): + if isinstance(config_dict[key], dict): + for k in config_dict[key]: + _lookup(config_dict[key], space_dict[key], k) + else: + if isinstance(space_dict[key], hpo.base.pyll.Apply) \ + and space_dict[key].name == "switch": + if len(space_dict[key].pos_args) > 0: + categories = [ + a.obj for a in space_dict[key].pos_args[1:] + if a.name == "literal" + ] + try: + idx = categories.index(config_dict[key]) + except ValueError as exc: + msg = f"Did not find category with value " \ + f"`{config_dict[key]}` in " \ + f"hyperopt parameter `{key}`. " + + if isinstance(config_dict[key], int): + msg += "In previous versions, a numerical " \ + "index was expected for categorical " \ + "values of `points_to_evaluate`, " \ + "but in ray>=1.2.0, the categorical " \ + "value is expected to be directly " \ + "provided. " + + msg += "Please make sure the specified category " \ + "is valid." + raise ValueError(msg) from exc + config_dict[key] = idx + + for k in config: + _lookup(config, self._space, k) + def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool: if self.domain: diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py index d2592dfe9..669114d9b 100644 --- a/python/ray/tune/suggest/nevergrad.py +++ b/python/ray/tune/suggest/nevergrad.py @@ -53,9 +53,9 @@ class NevergradSearch(Searcher): minimizing or maximizing the metric attribute. points_to_evaluate (list): Initial parameter suggestions to be run first. This is for when you already have some good parameters - you want hyperopt to run first to help the TPE algorithm - make better suggestions for future parameters. Needs to be - a list of dict of hyperopt-named variables. + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. use_early_stopped_trials: Deprecated. max_concurrent: Deprecated. @@ -113,8 +113,8 @@ class NevergradSearch(Searcher): space: Optional[Union[Dict, Parameter]] = None, metric: Optional[str] = None, mode: Optional[str] = None, - max_concurrent: Optional[int] = None, points_to_evaluate: Optional[List[Dict]] = None, + max_concurrent: Optional[int] = None, **kwargs): assert ng is not None, """Nevergrad must be installed! You can install Nevergrad with the command: @@ -204,6 +204,12 @@ class NevergradSearch(Searcher): raise ValueError("len(parameters_names) must match optimizer " "dimension for non-instrumented optimizers") + if self._points_to_evaluate: + # Nevergrad is LIFO, so we add the points to evaluate in reverse + # order. + for i in range(len(self._points_to_evaluate) - 1, -1, -1): + self._nevergrad_opt.suggest(self._points_to_evaluate[i]) + def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool: if self._nevergrad_opt or self._space: @@ -235,10 +241,6 @@ class NevergradSearch(Searcher): if len(self._live_trial_mapping) >= self.max_concurrent: return None - if self._points_to_evaluate is not None: - if len(self._points_to_evaluate) > 0: - point_to_evaluate = self._points_to_evaluate.pop(0) - self._nevergrad_opt.suggest(point_to_evaluate) suggested_config = self._nevergrad_opt.ask() self._live_trial_mapping[trial_id] = suggested_config diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py index 7076b1f59..f2947e895 100644 --- a/python/ray/tune/suggest/optuna.py +++ b/python/ray/tune/suggest/optuna.py @@ -62,6 +62,11 @@ class OptunaSearch(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. sampler (optuna.samplers.BaseSampler): Optuna sampler used to draw hyperparameter configurations. Defaults to ``TPESampler``. @@ -109,6 +114,7 @@ class OptunaSearch(Searcher): space: Optional[Union[Dict, List[Tuple]]] = None, metric: Optional[str] = None, mode: Optional[str] = None, + points_to_evaluate: Optional[List[Dict]] = None, sampler: Optional[BaseSampler] = None): assert ot is not None, ( "Optuna must be installed! Run `pip install optuna`.") @@ -128,6 +134,8 @@ class OptunaSearch(Searcher): self._space = space + self._points_to_evaluate = points_to_evaluate + self._study_name = "optuna" # Fixed study name for in-memory storage self._sampler = sampler or ot.samplers.TPESampler() assert isinstance(self._sampler, BaseSampler), \ @@ -188,12 +196,15 @@ class OptunaSearch(Searcher): ot_trial_id) ot_trial = self._ot_trials[trial_id] - # getattr will fetch the trial.suggest_ function on Optuna trials - params = { - args[0] if len(args) > 0 else kwargs["name"]: getattr( - ot_trial, fn)(*args, **kwargs) - for (fn, args, kwargs) in self._space - } + if self._points_to_evaluate: + params = self._points_to_evaluate.pop(0) + else: + # getattr will fetch the trial.suggest_ function on Optuna trials + params = { + args[0] if len(args) > 0 else kwargs["name"]: getattr( + ot_trial, fn)(*args, **kwargs) + for (fn, args, kwargs) in self._space + } return unflatten_dict(params) def on_trial_result(self, trial_id: str, result: Dict): @@ -215,7 +226,8 @@ class OptunaSearch(Searcher): def save(self, checkpoint_path: str): save_object = (self._storage, self._pruner, self._sampler, - self._ot_trials, self._ot_study) + self._ot_trials, self._ot_study, + self._points_to_evaluate) with open(checkpoint_path, "wb") as outputFile: pickle.dump(save_object, outputFile) @@ -223,7 +235,8 @@ class OptunaSearch(Searcher): with open(checkpoint_path, "rb") as inputFile: save_object = pickle.load(inputFile) self._storage, self._pruner, self._sampler, \ - self._ot_trials, self._ot_study = save_object + self._ot_trials, self._ot_study, \ + self._points_to_evaluate = save_object @staticmethod def convert_search_space(spec: Dict) -> List[Tuple]: diff --git a/python/ray/tune/suggest/skopt.py b/python/ray/tune/suggest/skopt.py index f8b8d9196..c329b94d5 100644 --- a/python/ray/tune/suggest/skopt.py +++ b/python/ray/tune/suggest/skopt.py @@ -1,3 +1,4 @@ +import copy import logging import pickle from typing import Dict, List, Optional, Tuple, Union @@ -21,7 +22,7 @@ logger = logging.getLogger(__name__) def _validate_warmstart(parameter_names: List[str], - points_to_evaluate: List[List], + points_to_evaluate: List[Union[List, Dict]], evaluated_rewards: List): if points_to_evaluate: if not isinstance(points_to_evaluate, list): @@ -29,10 +30,10 @@ def _validate_warmstart(parameter_names: List[str], "points_to_evaluate expected to be a list, got {}.".format( type(points_to_evaluate))) for point in points_to_evaluate: - if not isinstance(point, list): + if not isinstance(point, (dict, list)): raise TypeError( - "points_to_evaluate expected to include list, got {}.". - format(point)) + f"points_to_evaluate expected to include list or dict, " + f"got {point}.") if not len(point) == len(parameter_names): raise ValueError("Dim of point {}".format(point) + @@ -81,11 +82,11 @@ class SkOptSearch(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. - points_to_evaluate (list of lists): A list of points you'd like to run - first before sampling from the optimiser, e.g. these could be - parameter configurations you already know work well to help - the optimiser select good values. Each point is a list of the - parameters using the order definition given by parameter_names. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. evaluated_rewards (list): If you have previously evaluated the parameters passed in as points_to_evaluate you can avoid re-running those trials by passing in the reward attributes @@ -104,7 +105,16 @@ class SkOptSearch(Searcher): "height": tune.uniform(-100, 100) } - current_best_params = [[10, 0], [15, -20]] + current_best_params = [ + { + "width": 10, + "height": 0, + }, + { + "width": 15, + "height": -20, + } + ] skopt_search = SkOptSearch( metric="mean_loss", @@ -138,7 +148,7 @@ class SkOptSearch(Searcher): space: Union[List[str], Dict[str, Union[Tuple, List]]] = None, metric: Optional[str] = None, mode: Optional[str] = None, - points_to_evaluate: Optional[List[List]] = None, + points_to_evaluate: Optional[List[Dict]] = None, evaluated_rewards: Optional[List] = None, max_concurrent: Optional[int] = None, use_early_stopped_trials: Optional[bool] = None): @@ -182,7 +192,8 @@ class SkOptSearch(Searcher): self._parameter_names = list(space.keys()) self._parameter_ranges = space.values() - self._points_to_evaluate = points_to_evaluate + self._points_to_evaluate = copy.deepcopy(points_to_evaluate) + self._evaluated_rewards = evaluated_rewards self._skopt_opt = optimizer @@ -192,6 +203,16 @@ class SkOptSearch(Searcher): self._live_trial_mapping = {} def _setup_skopt(self): + if self._points_to_evaluate and isinstance(self._points_to_evaluate, + list): + if isinstance(self._points_to_evaluate[0], list): + # Keep backwards compatibility + self._points_to_evaluate = [ + dict(zip(self._parameter_names, point)) + for point in self._points_to_evaluate + ] + # Else: self._points_to_evaluate is already in correct format + _validate_warmstart(self._parameter_names, self._points_to_evaluate, self._evaluated_rewards) @@ -204,8 +225,9 @@ class SkOptSearch(Searcher): self._skopt_opt = sko.Optimizer(self._parameter_ranges) if self._points_to_evaluate and self._evaluated_rewards: - self._skopt_opt.tell(self._points_to_evaluate, - self._evaluated_rewards) + skopt_points = [[point[par] for par in self._parameter_names] + for point in self._points_to_evaluate] + self._skopt_opt.tell(skopt_points, self._evaluated_rewards) elif self._points_to_evaluate: self._initial_points = self._points_to_evaluate self._parameters = self._parameter_names @@ -254,12 +276,13 @@ class SkOptSearch(Searcher): if len(self._live_trial_mapping) >= self.max_concurrent: return None if self._initial_points: - suggested_config = self._initial_points[0] - del self._initial_points[0] + suggested_config = self._initial_points.pop(0) + skopt_config = [suggested_config[par] for par in self._parameters] else: - suggested_config = self._skopt_opt.ask() - self._live_trial_mapping[trial_id] = suggested_config - return unflatten_dict(dict(zip(self._parameters, suggested_config))) + skopt_config = self._skopt_opt.ask() + suggested_config = dict(zip(self._parameters, skopt_config)) + self._live_trial_mapping[trial_id] = skopt_config + return unflatten_dict(suggested_config) def on_trial_complete(self, trial_id: str, diff --git a/python/ray/tune/suggest/zoopt.py b/python/ray/tune/suggest/zoopt.py index 966a73f7d..f9e3d04bb 100644 --- a/python/ray/tune/suggest/zoopt.py +++ b/python/ray/tune/suggest/zoopt.py @@ -1,6 +1,6 @@ import copy import logging -from typing import Dict, Optional, Tuple +from typing import Dict, List, Optional, Tuple import ray import ray.cloudpickle as pickle @@ -11,7 +11,7 @@ from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils.util import unflatten_dict -from zoopt import ValueType +from zoopt import Solution, ValueType try: import zoopt @@ -119,6 +119,11 @@ class ZOOptSearch(Searcher): per default. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. + points_to_evaluate (list): Initial parameter suggestions to be run + first. This is for when you already have some good parameters + you want to run first to help the algorithm make better suggestions + for future parameters. Needs to be a list of dicts containing the + configurations. parallel_num (int): How many workers to parallel. Note that initial phase may start less workers than this number. More details can be found in zoopt package. @@ -132,6 +137,7 @@ class ZOOptSearch(Searcher): dim_dict: Optional[Dict] = None, metric: Optional[str] = None, mode: Optional[str] = None, + points_to_evaluate: Optional[List[Dict]] = None, **kwargs): assert zoopt is not None, "ZOOpt not found - please install zoopt " \ "by `pip install -U zoopt`." @@ -160,6 +166,9 @@ class ZOOptSearch(Searcher): self._metric_op = -1. elif mode == "min": self._metric_op = 1. + + self._points_to_evaluate = copy.deepcopy(points_to_evaluate) + self._live_trial_mapping = {} self._dim_keys = [] @@ -184,12 +193,22 @@ class ZOOptSearch(Searcher): self._dim_keys.append(k) _dim_list.append(self._dim_dict[k]) + init_samples = None + if self._points_to_evaluate: + logger.warning( + "`points_to_evaluate` seems to be ignored by ZOOpt.") + init_samples = [ + Solution(x=tuple(point[dim] for dim in self._dim_keys)) + for point in self._points_to_evaluate + ] dim = zoopt.Dimension2(_dim_list) - par = zoopt.Parameter(budget=self._budget) + par = zoopt.Parameter(budget=self._budget, init_samples=init_samples) if self._algo == "sracos" or self._algo == "asracos": from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune self.optimizer = SRacosTune( dimension=dim, parameter=par, **self.kwargs) + if init_samples: + self.optimizer.init_attribute() def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool: diff --git a/python/ray/tune/tests/_test_cluster_interrupt_searcher.py b/python/ray/tune/tests/_test_cluster_interrupt_searcher.py index 3e7e868b1..17edd250a 100644 --- a/python/ray/tune/tests/_test_cluster_interrupt_searcher.py +++ b/python/ray/tune/tests/_test_cluster_interrupt_searcher.py @@ -28,12 +28,12 @@ if __name__ == "__main__": { "width": 1, "height": 2, - "activation": 0 # Activation will be relu + "activation": "relu" # Activation will be relu }, { "width": 4, "height": 2, - "activation": 1 # Activation will be tanh + "activation": "tanh" # Activation will be tanh } ] algo = HyperOptSearch( diff --git a/python/ray/tune/tests/test_api.py b/python/ray/tune/tests/test_api.py index cfb1d3f6c..2a6f96868 100644 --- a/python/ray/tune/tests/test_api.py +++ b/python/ray/tune/tests/test_api.py @@ -17,6 +17,7 @@ from ray.tune import (DurableTrainable, Trainable, TuneError, Stopper, from ray.tune import register_env, register_trainable, run_experiments from ray.tune.schedulers import (TrialScheduler, FIFOScheduler, AsyncHyperBandScheduler) +from ray.tune.stopper import MaximumIterationStopper, TrialPlateauStopper from ray.tune.trial import Trial from ray.tune.result import (TIMESTEPS_TOTAL, DONE, HOSTNAME, NODE_IP, PID, EPISODES_TOTAL, TRAINING_ITERATION, @@ -556,6 +557,44 @@ class TrainableFunctionApiTest(unittest.TestCase): with self.assertRaises(TuneError): tune.run(train, stop=stop) + def testMaximumIterationStopper(self): + def train(config): + for i in range(10): + tune.report(it=i) + + stopper = MaximumIterationStopper(max_iter=6) + + out = tune.run(train, stop=stopper) + self.assertEqual(out.trials[0].last_result[TRAINING_ITERATION], 6) + + def testTrialPlateauStopper(self): + def train(config): + tune.report(10.0) + tune.report(11.0) + tune.report(12.0) + for i in range(10): + tune.report(20.0) + + # num_results = 4, no other constraints --> early stop after 7 + stopper = TrialPlateauStopper(metric="_metric", num_results=4) + + out = tune.run(train, stop=stopper) + self.assertEqual(out.trials[0].last_result[TRAINING_ITERATION], 7) + + # num_results = 4, grace period 9 --> early stop after 9 + stopper = TrialPlateauStopper( + metric="_metric", num_results=4, grace_period=9) + + out = tune.run(train, stop=stopper) + self.assertEqual(out.trials[0].last_result[TRAINING_ITERATION], 9) + + # num_results = 4, min_metric = 22 --> full 13 iterations + stopper = TrialPlateauStopper( + metric="_metric", num_results=4, metric_threshold=22.0, mode="max") + + out = tune.run(train, stop=stopper) + self.assertEqual(out.trials[0].last_result[TRAINING_ITERATION], 13) + def testCustomTrialDir(self): def train(config): for i in range(10): diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py index c297b8cdd..3a65799fc 100644 --- a/python/ray/tune/tests/test_progress_reporter.py +++ b/python/ray/tune/tests/test_progress_reporter.py @@ -73,7 +73,7 @@ tune.run_experiments({ "c": tune.grid_search(list(range(10))), }, }, -}, verbose=1, progress_reporter=reporter)""" +}, verbose=3, progress_reporter=reporter)""" EXPECTED_END_TO_END_START = """Number of trials: 1/30 (1 RUNNING) +---------------+----------+-------+-----+ @@ -160,6 +160,52 @@ EXPECTED_BEST_1 = "Current best trial: 00001 with metric_1=0.5 and " \ EXPECTED_BEST_2 = "Current best trial: 00004 with metric_1=2.0 and " \ "parameters={'a': 4}" +VERBOSE_EXP_OUT_1 = "Number of trials: 1/3 (1 RUNNING)" +VERBOSE_EXP_OUT_2 = "Number of trials: 3/3 (3 TERMINATED)" + +VERBOSE_TRIAL_NORM = "Trial train_xxxxx_00000 reported acc=5 with " + \ + """parameters={'do': 'complete'}. This trial completed. +Trial train_xxxxx_00001 reported _metric=6 with parameters={'do': 'once'}. +Trial train_xxxxx_00001 completed. Last result: _metric=6 +Trial train_xxxxx_00002 reported acc=7 with parameters={'do': 'twice'}. +Trial train_xxxxx_00002 reported acc=8 with parameters={'do': 'twice'}. """ + \ + "This trial completed." + +VERBOSE_TRIAL_DETAIL = """+-------------------+----------+-------+----------+ +| Trial name | status | loc | do | +|-------------------+----------+-------+----------| +| train_xxxxx_00000 | RUNNING | | complete | ++-------------------+----------+-------+----------+""" + +VERBOSE_CMD = """from ray import tune +import random +import numpy as np +import time + + +def train(config): + if config["do"] == "complete": + time.sleep(0.1) + tune.report(acc=5, done=True) + elif config["do"] == "once": + time.sleep(0.5) + tune.report(6) + else: + time.sleep(1.0) + tune.report(acc=7) + tune.report(acc=8) + +random.seed(1234) +np.random.seed(1234) + +tune.run( + train, + config={ + "do": tune.grid_search(["complete", "once", "twice"]) + },""" + +# Add "verbose=3)" etc + class ProgressReporterTest(unittest.TestCase): def mock_trial(self, status, i): @@ -294,12 +340,16 @@ class ProgressReporterTest(unittest.TestCase): trials.append(t) # One metric, two parameters prog1 = trial_progress_str( - trials, ["metric_1"], ["a", "b"], fmt="psql", max_rows=3) + trials, ["metric_1"], ["a", "b"], + fmt="psql", + max_rows=3, + force_table=True) print(prog1) assert prog1 == EXPECTED_RESULT_1 # No metric, all parameters - prog2 = trial_progress_str(trials, [], None, fmt="psql", max_rows=None) + prog2 = trial_progress_str( + trials, [], None, fmt="psql", max_rows=None, force_table=True) print(prog2) assert prog2 == EXPECTED_RESULT_2 @@ -310,7 +360,8 @@ class ProgressReporterTest(unittest.TestCase): "metric_2": "Metric 2" }, {"a": "A"}, fmt="psql", - max_rows=3) + max_rows=3, + force_table=True) print(prog3) assert prog3 == EXPECTED_RESULT_3 @@ -363,6 +414,64 @@ class ProgressReporterTest(unittest.TestCase): finally: del os.environ["_TEST_TUNE_TRIAL_UUID"] + def testVerboseReporting(self): + try: + os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx" + + verbose_0_cmd = VERBOSE_CMD + "verbose=0)" + output = run_string_as_driver(verbose_0_cmd) + try: + self.assertNotIn(VERBOSE_EXP_OUT_1, output) + self.assertNotIn(VERBOSE_EXP_OUT_2, output) + self.assertNotIn(VERBOSE_TRIAL_NORM, output) + self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) + except Exception: + print("*** BEGIN OUTPUT ***") + print(output) + print("*** END OUTPUT ***") + raise + + verbose_1_cmd = VERBOSE_CMD + "verbose=1)" + output = run_string_as_driver(verbose_1_cmd) + try: + self.assertIn(VERBOSE_EXP_OUT_1, output) + self.assertIn(VERBOSE_EXP_OUT_2, output) + self.assertNotIn(VERBOSE_TRIAL_NORM, output) + self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) + except Exception: + print("*** BEGIN OUTPUT ***") + print(output) + print("*** END OUTPUT ***") + raise + + verbose_2_cmd = VERBOSE_CMD + "verbose=2)" + output = run_string_as_driver(verbose_2_cmd) + try: + self.assertIn(VERBOSE_EXP_OUT_1, output) + self.assertIn(VERBOSE_EXP_OUT_2, output) + self.assertIn(VERBOSE_TRIAL_NORM, output) + self.assertNotIn(VERBOSE_TRIAL_DETAIL, output) + except Exception: + print("*** BEGIN OUTPUT ***") + print(output) + print("*** END OUTPUT ***") + raise + + verbose_3_cmd = VERBOSE_CMD + "verbose=3)" + output = run_string_as_driver(verbose_3_cmd) + try: + self.assertIn(VERBOSE_EXP_OUT_1, output) + self.assertIn(VERBOSE_EXP_OUT_2, output) + self.assertNotIn(VERBOSE_TRIAL_NORM, output) + self.assertIn(VERBOSE_TRIAL_DETAIL, output) + except Exception: + print("*** BEGIN OUTPUT ***") + print(output) + print("*** END OUTPUT ***") + raise + finally: + del os.environ["_TEST_TUNE_TRIAL_UUID"] + if __name__ == "__main__": import sys diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index 4f7b4a83d..d40900a6c 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -253,10 +253,14 @@ class SearchSpaceTest(unittest.TestCase): self.assertLess(config1["b"]["z"], 1e-2) searcher = BayesOptSearch() + invalid_config = {"a/b": tune.uniform(4.0, 8.0)} + with self.assertRaises(ValueError): searcher.set_search_properties("none", "max", invalid_config) + invalid_config = {"a": {"b/c": tune.uniform(4.0, 8.0)}} + with self.assertRaises(ValueError): searcher.set_search_properties("none", "max", invalid_config) @@ -373,7 +377,7 @@ class SearchSpaceTest(unittest.TestCase): config2 = searcher2.suggest("0") self.assertEqual(config1, config2) - self.assertLess(config2["point"], 1e-2) + self.assertLess(config2["b"]["z"], 1e-2) searcher = DragonflySearch() invalid_config = {"a/b": tune.uniform(4.0, 8.0)} @@ -388,7 +392,7 @@ class SearchSpaceTest(unittest.TestCase): analysis = tune.run( _mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] - self.assertLess(trial.config["point"], 1e-2) + self.assertLess(trial.config["b"]["z"], 1e-2) mixed_config = { "a": tune.uniform(5, 6), @@ -402,8 +406,8 @@ class SearchSpaceTest(unittest.TestCase): mode="max") config = searcher.suggest("0") - self.assertTrue(5 <= config["point"][0] <= 6) - self.assertTrue(8 <= config["point"][1] <= 9) + self.assertTrue(5 <= config["a"] <= 6) + self.assertTrue(8 <= config["b"] <= 9) def testConvertHyperOpt(self): from ray.tune.suggest.hyperopt import HyperOptSearch @@ -567,49 +571,6 @@ class SearchSpaceTest(unittest.TestCase): self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9) - def testNevergradBestParams(self): - from ray.tune.suggest.nevergrad import NevergradSearch - import nevergrad as ng - - config = { - "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), - "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), - "b": tune.sample.Integer(0, 5), - "c": tune.sample.Float(1e-4, 1e-1).loguniform() - } - - best_params = [{ - "metric": 1, - "a": "t1", - "b": 1, - "c": 1e-1 - }, { - "metric": 2, - "a": "t2", - "b": 2, - "c": 1e-2 - }] - - searcher = NevergradSearch( - optimizer=ng.optimizers.OnePlusOne, points_to_evaluate=best_params) - analysis = tune.run( - _mock_objective, - config=config, - metric="metric", - mode="max", - search_alg=searcher, - num_samples=5) - - for i in range(len(best_params)): - trial_config = analysis.trials[i].config - trial_config_dict = { - "metric": trial_config["metric"], - "a": trial_config["a"], - "b": trial_config["b"], - "c": trial_config["c"] - } - self.assertDictEqual(trial_config_dict, best_params[i]) - def testConvertOptuna(self): from ray.tune.suggest.optuna import OptunaSearch, param from optuna.samplers import RandomSampler @@ -780,6 +741,136 @@ class SearchSpaceTest(unittest.TestCase): self.assertTrue(5 <= config["a"] <= 6) self.assertTrue(8 <= config["b"] <= 9) + def _testPointsToEvaluate(self, cls, config, **kwargs): + points_to_evaluate = [{k: v.sample() + for k, v in config.items()} for _ in range(2)] + print(f"Points to evaluate: {points_to_evaluate}") + searcher = cls(points_to_evaluate=points_to_evaluate, **kwargs) + + analysis = tune.run( + _mock_objective, + config=config, + metric="metric", + mode="max", + search_alg=searcher, + num_samples=5) + + for i in range(len(points_to_evaluate)): + trial_config = analysis.trials[i].config + trial_config_dict = { + "metric": trial_config["metric"], + "a": trial_config["a"], + "b": trial_config["b"], + "c": trial_config["c"] + } + self.assertDictEqual(trial_config_dict, points_to_evaluate[i]) + + def testPointsToEvaluateAx(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.ax import AxSearch + return self._testPointsToEvaluate(AxSearch, config) + + def testPointsToEvaluateBayesOpt(self): + config = { + "metric": tune.sample.Float(10, 20).uniform(), + "a": tune.sample.Float(-30, -20).uniform(), + "b": tune.sample.Float(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.bayesopt import BayesOptSearch + return self._testPointsToEvaluate(BayesOptSearch, config) + + def testPointsToEvaluateBOHB(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.bohb import TuneBOHB + return self._testPointsToEvaluate(TuneBOHB, config) + + def testPointsToEvaluateDragonfly(self): + config = { + "metric": tune.sample.Float(10, 20).uniform(), + "a": tune.sample.Float(-30, -20).uniform(), + "b": tune.sample.Float(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.dragonfly import DragonflySearch + return self._testPointsToEvaluate( + DragonflySearch, config, domain="euclidean", optimizer="bandit") + + def testPointsToEvaluateHyperOpt(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.hyperopt import HyperOptSearch + return self._testPointsToEvaluate(HyperOptSearch, config) + + def testPointsToEvaluateNevergrad(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.nevergrad import NevergradSearch + import nevergrad as ng + return self._testPointsToEvaluate( + NevergradSearch, config, optimizer=ng.optimizers.OnePlusOne) + + def testPointsToEvaluateOptuna(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.optuna import OptunaSearch + return self._testPointsToEvaluate(OptunaSearch, config) + + def testPointsToEvaluateSkOpt(self): + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).loguniform() + } + + from ray.tune.suggest.skopt import SkOptSearch + return self._testPointsToEvaluate(SkOptSearch, config) + + def testPointsToEvaluateZoOpt(self): + # https://github.com/polixir/ZOOpt/issues/5 + self.skipTest("ZoOpt currently ignores initial points. This test " + "will be enabled after this has been fixed.") + config = { + "metric": tune.sample.Categorical([1, 2, 3, 4]).uniform(), + "a": tune.sample.Categorical(["t1", "t2", "t3", "t4"]).uniform(), + "b": tune.sample.Integer(0, 5), + "c": tune.sample.Float(1e-4, 1e-1).uniform() + } + + from ray.tune.suggest.zoopt import ZOOptSearch + return self._testPointsToEvaluate( + ZOOptSearch, config, budget=10, parallel_num=8) + if __name__ == "__main__": import pytest diff --git a/python/ray/tune/trial.py b/python/ray/tune/trial.py index b775639d3..2e9465c83 100644 --- a/python/ray/tune/trial.py +++ b/python/ray/tune/trial.py @@ -16,7 +16,6 @@ from ray.tune.checkpoint_manager import Checkpoint, CheckpointManager # NOTE(rkn): We import ray.tune.registry here instead of importing the names we # need because there are cyclic imports that may cause specific names to not # have been defined yet. See https://github.com/ray-project/ray/issues/1716. -from ray.tune.logger import pretty_print from ray.tune.registry import get_trainable_cls, validate_trainable from ray.tune.result import DEFAULT_RESULTS_DIR, DONE, TRAINING_ITERATION from ray.tune.resources import Resources, json_to_resources, resources_to_json @@ -230,7 +229,6 @@ class Trial: or not len(self.log_to_file) == 2: self.log_to_file = (None, None) - self.verbose = True self.max_failures = max_failures # Local trial state that is updated during the run @@ -480,11 +478,7 @@ class Trial: def update_last_result(self, result, terminate=False): if self.experiment_tag: result.update(experiment_tag=self.experiment_tag) - if self.verbose and (terminate or time.time() - self.last_debug > - DEBUG_PRINT_INTERVAL): - print("Result for {}:".format(self)) - print(" {}".format(pretty_print(result).replace("\n", "\n "))) - self.last_debug = time.time() + self.set_location(Location(result.get("node_ip"), result.get("pid"))) self.last_result = result self.last_update_time = time.time() @@ -527,9 +521,6 @@ class Trial: def get_trainable_cls(self): return get_trainable_cls(self.trainable_name) - def set_verbose(self, verbose): - self.verbose = verbose - def is_finished(self): return self.status in [Trial.ERROR, Trial.TERMINATED] diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 7b28fc144..abf41ee77 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -19,6 +19,7 @@ from ray.tune.trial import Checkpoint, Trial from ray.tune.schedulers import FIFOScheduler, TrialScheduler from ray.tune.suggest import BasicVariantGenerator from ray.tune.utils import warn_if_slow, flatten_dict, env_integer +from ray.tune.utils.log import Verbosity, has_verbosity from ray.tune.utils.serialization import TuneFunctionDecoder, \ TuneFunctionEncoder from ray.tune.web_server import TuneServer @@ -78,8 +79,6 @@ class TrialRunner: If fail_fast='raise' provided, Tune will automatically raise the exception received by the Trainable. fail_fast='raise' can easily leak resources and should be used with caution. - verbose (bool): Flag for verbosity. If False, trial results - will not be output. checkpoint_period (int): Trial runner checkpoint periodicity in seconds. Defaults to 10. trial_executor (TrialExecutor): Defaults to RayTrialExecutor. @@ -102,7 +101,6 @@ class TrialRunner: resume=False, server_port=None, fail_fast=False, - verbose=True, checkpoint_period=None, trial_executor=None, callbacks=None, @@ -135,7 +133,6 @@ class TrialRunner: else: raise ValueError("fail_fast must be one of {bool, RAISE}. " f"Got {self._fail_fast}.") - self._verbose = verbose self._server = None self._server_port = server_port @@ -165,7 +162,7 @@ class TrialRunner: self.resume(run_errored_only=errored_only) self._resumed = True except Exception as e: - if self._verbose: + if has_verbosity(Verbosity.V3_TRIAL_DETAILS): logger.error(str(e)) logger.exception("Runner restore failed.") if self._fail_fast: @@ -360,8 +357,7 @@ class TrialRunner: trials=self._trials, trial=next_trial) elif self.trial_executor.get_running_trials(): - with warn_if_slow("process_events"): - self._process_events() # blocking + self._process_events() # blocking else: self.trial_executor.on_no_available_trials(self) @@ -405,7 +401,6 @@ class TrialRunner: Args: trial (Trial): Trial to queue. """ - trial.set_verbose(self._verbose) self._trials.append(trial) with warn_if_slow("scheduler.on_trial_add"): self._scheduler_alg.on_trial_add(self, trial) @@ -565,6 +560,8 @@ class TrialRunner: with warn_if_slow("scheduler.on_trial_result"): decision = self._scheduler_alg.on_trial_result( self, trial, flat_result) + if decision == TrialScheduler.STOP: + result.update(done=True) with warn_if_slow("search_alg.on_trial_result"): self._search_alg.on_trial_result(trial.trial_id, flat_result) @@ -583,7 +580,6 @@ class TrialRunner: iteration=self._iteration, trials=self._trials, trial=trial) - result.update(done=True) if not is_duplicate: trial.update_last_result( diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py index 3566b95ff..fe26e12e5 100644 --- a/python/ray/tune/tune.py +++ b/python/ray/tune/tune.py @@ -18,6 +18,7 @@ from ray.tune.syncer import wait_for_sync, set_sync_periods, \ from ray.tune.trial_runner import TrialRunner from ray.tune.progress_reporter import CLIReporter, JupyterNotebookReporter from ray.tune.schedulers import FIFOScheduler +from ray.tune.utils.log import Verbosity, has_verbosity, set_verbosity logger = logging.getLogger(__name__) @@ -70,7 +71,7 @@ def run( checkpoint_score_attr=None, checkpoint_freq=0, checkpoint_at_end=False, - verbose=2, + verbose=Verbosity.V3_TRIAL_DETAILS, progress_reporter=None, log_to_file=False, trial_name_creator=None, @@ -188,8 +189,9 @@ def run( checkpoint_at_end (bool): Whether to checkpoint at the end of the experiment regardless of the checkpoint_freq. Default is False. This has no effect when using the Functional Training API. - verbose (int): 0, 1, or 2. Verbosity mode. 0 = silent, - 1 = only status updates, 2 = status and trial results. + verbose (Union[int, Verbosity]): 0, 1, 2, or 3. Verbosity mode. + 0 = silent, 1 = only status updates, 2 = status and brief trial + results, 3 = status and detailed trial results. Defaults to 3. progress_reporter (ProgressReporter): Progress reporter for reporting intermediate experiment progress. Defaults to CLIReporter if running in command-line, or JupyterNotebookReporter if running in @@ -281,6 +283,8 @@ def run( "The `mode` parameter passed to `tune.run()` has to be one of " "['min', 'max']") + set_verbosity(verbose) + config = config or {} sync_config = sync_config or SyncConfig() set_sync_periods(sync_config) @@ -353,9 +357,9 @@ def run( "own `metric` and `mode` parameters. Either remove the arguments " "from your scheduler or from your call to `tune.run()`") - # Create logger and syncer callbacks + # Create syncer callbacks callbacks = create_default_callbacks( - callbacks, sync_config, loggers=loggers) + callbacks, sync_config, metric=metric, loggers=loggers) runner = TrialRunner( search_alg=search_alg, @@ -366,7 +370,6 @@ def run( stopper=experiments[0].stopper, resume=resume, server_port=server_port, - verbose=bool(verbose > 1), fail_fast=fail_fast, trial_executor=trial_executor, callbacks=callbacks, @@ -380,7 +383,8 @@ def run( if progress_reporter is None: if IS_NOTEBOOK: - progress_reporter = JupyterNotebookReporter(overwrite=verbose < 2) + progress_reporter = JupyterNotebookReporter( + overwrite=not has_verbosity(Verbosity.V2_TRIAL_NORM)) else: progress_reporter = CLIReporter() @@ -413,7 +417,7 @@ def run( tune_start = time.time() while not runner.is_finished(): runner.step() - if verbose: + if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter) tune_taken = time.time() - tune_start @@ -422,7 +426,7 @@ def run( except Exception as e: logger.warning(f"Trial Runner checkpointing failed: {str(e)}") - if verbose: + if has_verbosity(Verbosity.V1_EXPERIMENT): _report_progress(runner, progress_reporter, done=True) wait_for_sync() @@ -440,8 +444,9 @@ def run( logger.error("Trials did not complete: %s", incomplete_trials) all_taken = time.time() - all_start - logger.info(f"Total run time: {all_taken:.2f} seconds " - f"({tune_taken:.2f} seconds for the tuning loop).") + if has_verbosity(Verbosity.V1_EXPERIMENT): + logger.info(f"Total run time: {all_taken:.2f} seconds " + f"({tune_taken:.2f} seconds for the tuning loop).") trials = runner.get_trials() return ExperimentAnalysis( @@ -454,7 +459,7 @@ def run( def run_experiments(experiments, scheduler=None, server_port=None, - verbose=2, + verbose=Verbosity.V3_TRIAL_DETAILS, progress_reporter=None, resume=False, queue_trials=False, diff --git a/python/ray/tune/utils/callback.py b/python/ray/tune/utils/callback.py index b2f5fcf71..d54bd83ea 100644 --- a/python/ray/tune/utils/callback.py +++ b/python/ray/tune/utils/callback.py @@ -4,6 +4,7 @@ import logging import os from ray.tune.callback import Callback +from ray.tune.progress_reporter import TrialProgressCallback from ray.tune.syncer import SyncConfig, detect_sync_to_driver from ray.tune.logger import CSVLoggerCallback, CSVLogger, LoggerCallback, \ JsonLoggerCallback, JsonLogger, LegacyLoggerCallback, Logger, \ @@ -15,14 +16,44 @@ logger = logging.getLogger(__name__) def create_default_callbacks(callbacks: Optional[List[Callback]], sync_config: SyncConfig, - loggers: Optional[List[Logger]]): + loggers: Optional[List[Logger]], + metric: Optional[str] = None): + """Create default callbacks for `tune.run()`. + This function takes a list of existing callbacks and adds default + callbacks to it. + + Specifically, three kinds of callbacks will be added: + + 1. Loggers. Ray Tune's experiment analysis relies on CSV and JSON logging. + 2. Syncer. Ray Tune synchronizes logs and checkpoint between workers and + the head node. + 2. Trial progress reporter. For reporting intermediate progress, like trial + results, Ray Tune uses a callback. + + These callbacks will only be added if they don't already exist, i.e. if + they haven't been passed (and configured) by the user. A notable case + is when a Logger is passed, which is not a CSV or JSON logger - then + a CSV and JSON logger will still be created. + + Lastly, this function will ensure that the Syncer callback comes after all + Logger callbacks, to ensure that the most up-to-date logs and checkpoints + are synced across nodes. + + """ callbacks = callbacks or [] has_syncer_callback = False has_csv_logger = False has_json_logger = False has_tbx_logger = False + has_trial_progress_callback = any( + isinstance(c, TrialProgressCallback) for c in callbacks) + + if not has_trial_progress_callback: + trial_progress_callback = TrialProgressCallback(metric=metric) + callbacks.append(trial_progress_callback) + # Track syncer obj/index to move callback after loggers last_logger_index = None syncer_index = None diff --git a/python/ray/tune/utils/log.py b/python/ray/tune/utils/log.py new file mode 100644 index 000000000..a62eb2b66 --- /dev/null +++ b/python/ray/tune/utils/log.py @@ -0,0 +1,34 @@ +from enum import Enum +from typing import Union + + +class Verbosity(Enum): + V0_MINIMAL = 0 + V1_EXPERIMENT = 1 + V2_TRIAL_NORM = 2 + V3_TRIAL_DETAILS = 3 + + def __int__(self): + return self.value + + +verbosity: Union[int, Verbosity] = Verbosity.V3_TRIAL_DETAILS + + +def set_verbosity(level: Union[int, Verbosity]): + global verbosity + + if isinstance(level, int): + verbosity = Verbosity(level) + else: + verbosity = verbosity + + +def has_verbosity(level: Union[int, Verbosity]) -> bool: + """Return True if passed level exceeds global verbosity level.""" + global verbosity + + log_level = int(level) + verbosity_level = int(verbosity) + + return verbosity_level >= log_level diff --git a/python/ray/tune/utils/mock.py b/python/ray/tune/utils/mock.py index 1ec925519..cc92fae26 100644 --- a/python/ray/tune/utils/mock.py +++ b/python/ray/tune/utils/mock.py @@ -102,11 +102,11 @@ class FailureInjectorCallback(Callback): """Adds random failure injection to the TrialExecutor.""" def __init__(self, - config_path="/home/ubuntu/ray_bootstrap_config.yaml", + config_path="~/ray_bootstrap_config.yaml", probability=0.1, disable=False): self.probability = probability - self.config_path = config_path + self.config_path = os.path.expanduser(config_path) self.disable = disable def on_step_begin(self, **info): diff --git a/python/ray/util/__init__.py b/python/ray/util/__init__.py index be876c649..2bead8d18 100644 --- a/python/ray/util/__init__.py +++ b/python/ray/util/__init__.py @@ -10,5 +10,5 @@ from ray.util import rpdb as pdb __all__ = [ "ActorPool", "disable_log_once_globally", "enable_periodic_logging", "iter", "log_once", "pdb", "placement_group", "placement_group_table", - "remove_placement_group", "inspect_serializability" + "remove_placement_group", "inspect_serializability", "collective" ] diff --git a/python/ray/util/collective/__init__.py b/python/ray/util/collective/__init__.py new file mode 100644 index 000000000..68fcb78d4 --- /dev/null +++ b/python/ray/util/collective/__init__.py @@ -0,0 +1,9 @@ +from .collective import nccl_available, mpi_available, is_group_initialized, \ + init_collective_group, destroy_collective_group, get_rank, \ + get_world_size, allreduce, barrier + +__all__ = [ + "nccl_available", "mpi_available", "is_group_initialized", + "init_collective_group", "destroy_collective_group", "get_rank", + "get_world_size", "allreduce", "barrier" +] diff --git a/python/ray/util/collective/collective.py b/python/ray/util/collective/collective.py new file mode 100644 index 000000000..343487e71 --- /dev/null +++ b/python/ray/util/collective/collective.py @@ -0,0 +1,275 @@ +"""APIs exposed under the namespace ray.util.collective.""" +import logging + +import numpy as np +import ray +from ray.util.collective import types +from ray.util.collective.const import get_nccl_store_name + +_MPI_AVAILABLE = False +_NCCL_AVAILABLE = True + +# try: +# from ray.util.collective.collective_group.mpi_collective_group \ +# import MPIGroup +# except ImportError: +# _MPI_AVAILABLE = False +try: + from ray.util.collective.collective_group import NCCLGroup + from ray.util.collective.collective_group import nccl_util +except ImportError: + _NCCL_AVAILABLE = False + +logger = logging.getLogger(__name__) + + +def nccl_available(): + return _NCCL_AVAILABLE + + +def mpi_available(): + return _MPI_AVAILABLE + + +class GroupManager(object): + """ + Use this class to manage the collective groups we created so far. + + Each process will have an instance of `GroupManager`. Each process + could belong to multiple collective groups. The membership information + and other metadata are stored in the global `_group_mgr` object. + """ + + def __init__(self): + self._name_group_map = {} + self._group_name_map = {} + + def create_collective_group(self, backend, world_size, rank, group_name): + """ + The entry to create new collective groups and register in the manager. + + Put the registration and the group information into the manager + metadata as well. + """ + backend = types.Backend(backend) + if backend == types.Backend.MPI: + raise NotImplementedError() + elif backend == types.Backend.NCCL: + # create the ncclUniqueID + if rank == 0: + # availability has been checked before entering here. + group_uid = nccl_util.get_nccl_unique_id() + store_name = get_nccl_store_name(group_name) + # Avoid a potential circular dependency in ray/actor.py + from ray.util.collective.util import NCCLUniqueIDStore + store = NCCLUniqueIDStore.options( + name=store_name, lifetime="detached").remote(store_name) + ray.wait([store.set_id.remote(group_uid)]) + + logger.debug("creating NCCL group: '{}'".format(group_name)) + g = NCCLGroup(world_size, rank, group_name) + self._name_group_map[group_name] = g + self._group_name_map[g] = group_name + return self._name_group_map[group_name] + + def is_group_exist(self, group_name): + return group_name in self._name_group_map + + def get_group_by_name(self, group_name): + """Get the collective group handle by its name.""" + if not self.is_group_exist(group_name): + logger.warning( + "The group '{}' is not initialized.".format(group_name)) + return None + return self._name_group_map[group_name] + + def destroy_collective_group(self, group_name): + """Group destructor.""" + if not self.is_group_exist(group_name): + logger.warning("The group '{}' does not exist.".format(group_name)) + return + + # release the collective group resource + g = self._name_group_map[group_name] + rank = g.rank + backend = g.backend() + + # clean up the dicts + del self._group_name_map[g] + del self._name_group_map[group_name] + if backend == types.Backend.NCCL: + # release the named actor + if rank == 0: + store_name = get_nccl_store_name(group_name) + store = ray.get_actor(store_name) + ray.wait([store.__ray_terminate__.remote()]) + ray.kill(store) + # Release the communicator resources + g.destroy_group() + + +_group_mgr = GroupManager() + + +def is_group_initialized(group_name): + """Check if the group is initialized in this process by the group name.""" + return _group_mgr.is_group_exist(group_name) + + +def init_collective_group(world_size: int, + rank: int, + backend=types.Backend.NCCL, + group_name: str = "default"): + """ + Initialize a collective group inside an actor process. + + Args: + world_size (int): the total number of processed in the group. + rank (int): the rank of the current process. + backend: the CCL backend to use, NCCL or MPI. + group_name (str): the name of the collective group. + + Returns: + None + """ + _check_inside_actor() + backend = types.Backend(backend) + _check_backend_availability(backend) + global _group_mgr + # TODO(Hao): implement a group auto-counter. + if not group_name: + raise ValueError("group_name '{}' needs to be a string." + .format(group_name)) + + if _group_mgr.is_group_exist(group_name): + raise RuntimeError("Trying to initialize a group twice.") + + assert (world_size > 0) + assert (rank >= 0) + assert (rank < world_size) + _group_mgr.create_collective_group(backend, world_size, rank, group_name) + + +def destroy_collective_group(group_name: str = "default") -> None: + """Destroy a collective group given its group name.""" + _check_inside_actor() + global _group_mgr + _group_mgr.destroy_collective_group(group_name) + + +def get_rank(group_name: str = "default") -> int: + """ + Return the rank of this process in the given group. + + Args: + group_name (str): the name of the group to query + + Returns: + the rank of this process in the named group, + -1 if the group does not exist or the process does + not belong to the group. + """ + _check_inside_actor() + if not is_group_initialized(group_name): + return -1 + g = _group_mgr.get_group_by_name(group_name) + return g.rank + + +def get_world_size(group_name="default") -> int: + """ + Return the size of the collective gropu with the given name. + + Args: + group_name: the name of the group to query + + Returns: + The world size of the collective group， + -1 if the group does not exist or the process does + not belong to the group. + """ + _check_inside_actor() + if not is_group_initialized(group_name): + return -1 + g = _group_mgr.get_group_by_name(group_name) + return g.world_size + + +def allreduce(tensor, group_name: str, op=types.ReduceOp.SUM): + """ + Collective allreduce the tensor across the group with name group_name. + + Args: + tensor: the tensor to be all-reduced on this process. + group_name (str): the collective group name to perform allreduce. + op: The reduce operation. + + Returns: + None + """ + _check_single_tensor_input(tensor) + g = _check_and_get_group(group_name) + opts = types.AllReduceOptions + opts.reduceOp = op + g.allreduce(tensor, opts) + + +def barrier(group_name): + """ + Barrier all processes in the collective group. + + Args: + group_name (str): the name of the group to barrier. + + Returns: + None + """ + g = _check_and_get_group(group_name) + g.barrier() + + +def _check_and_get_group(group_name): + """Check the existence and return the group handle.""" + _check_inside_actor() + if not is_group_initialized(group_name): + raise RuntimeError("The collective group '{}' is not " + "initialized in the process.".format(group_name)) + g = _group_mgr.get_group_by_name(group_name) + return g + + +def _check_backend_availability(backend: types.Backend): + """Check whether the backend is available.""" + if backend == types.Backend.MPI: + if not mpi_available(): + raise RuntimeError("MPI is not available.") + elif backend == types.Backend.NCCL: + # expect some slowdown at the first call + # as I defer the import to invocation. + if not nccl_available(): + raise RuntimeError("NCCL is not available.") + + +def _check_single_tensor_input(tensor): + """Check if the tensor is with a supported type.""" + if isinstance(tensor, np.ndarray): + return + if types.cupy_available(): + if isinstance(tensor, types.cp.ndarray): + return + if types.torch_available(): + if isinstance(tensor, types.th.Tensor): + return + raise RuntimeError("Unrecognized tensor type '{}'. Supported types are: " + "np.ndarray, torch.Tensor, cupy.ndarray.".format( + type(tensor))) + + +def _check_inside_actor(): + """Check if currently it is inside a Ray actor/task.""" + worker = ray.worker.global_worker + if worker.mode == ray.WORKER_MODE: + return + else: + raise RuntimeError("The collective APIs shall be only used inside " + "a Ray actor or task.") diff --git a/python/ray/util/collective/collective_group/__init__.py b/python/ray/util/collective/collective_group/__init__.py new file mode 100644 index 000000000..c8ecc463e --- /dev/null +++ b/python/ray/util/collective/collective_group/__init__.py @@ -0,0 +1,3 @@ +from .nccl_collective_group import NCCLGroup + +__all__ = ["NCCLGroup"] diff --git a/python/ray/util/collective/collective_group/base_collective_group.py b/python/ray/util/collective/collective_group/base_collective_group.py new file mode 100644 index 000000000..a3f54fa26 --- /dev/null +++ b/python/ray/util/collective/collective_group/base_collective_group.py @@ -0,0 +1,52 @@ +"""Abstract class for collective groups.""" +from abc import ABCMeta +from abc import abstractmethod + +from ray.util.collective.types import AllReduceOptions, BarrierOptions + + +class BaseGroup(metaclass=ABCMeta): + def __init__(self, world_size, rank, group_name): + """ + Init the process group with basic information. + + Args: + world_size (int): The total number of processes in the group. + rank (int): The rank of the current process. + group_name (str): The group name. + """ + self._world_size = world_size + self._rank = rank + self._group_name = group_name + + @property + def rank(self): + """Return the rank of the current process.""" + return self._rank + + @property + def world_size(self): + """Return the number of processes in this group.""" + return self._world_size + + @property + def group_name(self): + """Return the group name of this group.""" + return self._group_name + + def destroy_group(self): + """GC the communicators.""" + pass + + @classmethod + def backend(cls): + """The backend of this collective group.""" + raise NotImplementedError() + + @abstractmethod + def allreduce(self, tensor, allreduce_options=AllReduceOptions()): + raise NotImplementedError() + + @abstractmethod + def barrier(self, barrier_options=BarrierOptions()): + raise NotImplementedError() diff --git a/python/ray/util/collective/collective_group/mpi_collective_group.py b/python/ray/util/collective/collective_group/mpi_collective_group.py new file mode 100644 index 000000000..e045ac716 --- /dev/null +++ b/python/ray/util/collective/collective_group/mpi_collective_group.py @@ -0,0 +1,5 @@ +"""Implementation of the MPI collective group.""" +try: + import mpi4py # noqa: F401 +except ImportError: + raise diff --git a/python/ray/util/collective/collective_group/nccl_collective_group.py b/python/ray/util/collective/collective_group/nccl_collective_group.py new file mode 100644 index 000000000..31412b5a4 --- /dev/null +++ b/python/ray/util/collective/collective_group/nccl_collective_group.py @@ -0,0 +1,219 @@ +import logging +import datetime +import time + +import ray +import cupy + +from ray.util.collective.collective_group import nccl_util +from ray.util.collective.collective_group.base_collective_group \ + import BaseGroup +from ray.util.collective.types import AllReduceOptions, \ + BarrierOptions, Backend +from ray.util.collective.const import get_nccl_store_name + +logger = logging.getLogger(__name__) + +# TODO(Hao): +# (1) stream management, instead of using the default stream, +# using a dedicate stream +# (2) communicator management and support num_gpus > 2 per actor. + + +class Rendezvous: + """ + A rendezvous class for different actor/task processes to meet. + + To initialize an NCCL collective communication group, different + actors/tasks spawned in Ray in a collective group needs to meet + each other to synchronize the NCCLUniqueID. This class guarantees + they meet via the NCCLUniqueIDStore, initialized on the rank=0 + process. + + Args: + group_name (str): the unique user-specified group name. + """ + + def __init__(self, group_name): + if not group_name: + raise ValueError("Invalid group name.") + self._group_name = group_name + self._store_name = None + self._store = None + + def meet(self, timeout_s=180): + """ + Meet at the named actor store. + + Args: + timeout_s: timeout in seconds. + + Return: + None + """ + if timeout_s <= 0: + raise ValueError("The 'timeout' argument must be positive. " + "Got '{}'.".format(timeout_s)) + self._store_name = get_nccl_store_name(self._group_name) + timeout_delta = datetime.timedelta(seconds=timeout_s) + elapsed = datetime.timedelta(seconds=0) + start_time = datetime.datetime.now() + while elapsed < timeout_delta: + try: + logger.debug("Trying to meet at the store '{}'".format( + self._store_name)) + self._store = ray.get_actor(self._store_name) + except ValueError: + logger.debug("Failed to meet at the store '{}'." + "Trying again...".format(self._store_name)) + time.sleep(1) + elapsed = datetime.datetime.now() - start_time + continue + logger.debug("Successful rendezvous!") + break + if not self._store: + raise RuntimeError("Unable to meet other processes " + "at the rendezvous store.") + + @property + def store(self): + return self._store + + def get_nccl_id(self, timeout_s=180): + """ + Get the NCCLUniqueID from the store through Ray. + + Args: + timeout_s: timeout in seconds. + Return: + str: the NCCLUniqueID if successful. + """ + if not self._store: + raise ValueError("Rendezvous store is not setup.") + uid = None + timeout_delta = datetime.timedelta(seconds=timeout_s) + elapsed = datetime.timedelta(seconds=0) + start_time = datetime.datetime.now() + while elapsed < timeout_delta: + uid = ray.get(self._store.get_id.remote()) + if not uid: + time.sleep(1) + elapsed = datetime.datetime.now() - start_time + continue + break + if not uid: + raise RuntimeError( + "Unable to get the NCCLUniqueID from the store.") + return uid + + +class NCCLGroup(BaseGroup): + def __init__(self, world_size, rank, group_name): + """Init an NCCL collective group.""" + super(NCCLGroup, self).__init__(world_size, rank, group_name) + self._nccl_uid = None + + # TODO(Hao): change this to a be a cache + self._nccl_comm = None + + if nccl_util.get_nccl_build_version() < 2000: + raise RuntimeError("NCCL in Ray requires NCCL >= 2.0.") + # TODO(Hao): check version here + if nccl_util.get_nccl_runtime_version() < 2704: + logger.warning("NCCL send/recv calls requires NCCL>=2.7.4") + + self._rendezvous = Rendezvous(self.group_name) + self._rendezvous.meet() + + # Setup the nccl uid using the store + self._init_nccl_unique_id() + + # Setup a tensor for barrier calls + self._barrier_tensor = cupy.array([1]) + + def _init_nccl_unique_id(self): + """ + Init the NCCL unique ID required for setting up NCCL communicator. + + """ + self._nccl_uid = self._rendezvous.get_nccl_id() + + @property + def nccl_uid(self): + return self._nccl_uid + + def destroy_group(self): + """ + Destroy the group and release the NCCL communicators safely. + + """ + if self._nccl_comm is not None: + self.barrier() + # We also need a barrier call here. + stream = self._get_cuda_stream() + stream.synchronize() + # destroy the communicator + self._nccl_comm.destroy() + self._nccl_comm = None + super(NCCLGroup, self).destroy_group() + + @classmethod + def backend(cls): + return Backend.NCCL + + def allreduce(self, tensor, allreduce_options=AllReduceOptions()): + """ + AllReduce a list of tensors following options. + + Args: + tensor: the tensor to be reduced, each tensor locates on a GPU + allreduce_options: + + Returns: + """ + # obtain the communicator + comm = self._get_nccl_communicator() + # obtain the stream: using default stream by now + # TODO(Hao): implement a simple stream manager here + stream = self._get_cuda_stream() + + dtype = nccl_util.get_nccl_tensor_dtype(tensor) + ptr = nccl_util.get_tensor_ptr(tensor) + n_elems = nccl_util.get_tensor_n_elements(tensor) + reduce_op = nccl_util.get_nccl_reduce_op(allreduce_options.reduceOp) + + # in-place allreduce + comm.allReduce(ptr, ptr, n_elems, dtype, reduce_op, stream.ptr) + + def barrier(self, barrier_options=BarrierOptions()): + """ + Blocks until all processes reach this barrier. + + Args: + barrier_options: + + Returns: + """ + self.allreduce(self._barrier_tensor) + + def _get_nccl_communicator(self): + """ + Create or use a cached NCCL communicator for the collective task. + + """ + # TODO(Hao): later change this to use device keys and query from cache. + # TODO(Hao): implement a thin wrapper + if not self._nccl_comm: + self._nccl_comm = nccl_util.create_nccl_communicator( + self.world_size, self.nccl_uid, self.rank) + return self._nccl_comm + + @staticmethod + def _get_cuda_stream(): + """Obtain an idle stream from a stream pool for the collective task.""" + # TODO: implement a simple stream manager. + return cupy.cuda.Stream.null + + # def _collective_call(self, *args): + # """Private method to encapsulate all collective calls""" + # pass diff --git a/python/ray/util/collective/collective_group/nccl_util.py b/python/ray/util/collective/collective_group/nccl_util.py new file mode 100644 index 000000000..4d2fc456f --- /dev/null +++ b/python/ray/util/collective/collective_group/nccl_util.py @@ -0,0 +1,117 @@ +"""Code to wrap some NCCL API calls.""" +import numpy +try: + import cupy + from cupy.cuda import nccl + from cupy.cuda.nccl import get_version + from cupy.cuda.nccl import get_build_version + from cupy.cuda.nccl import NcclCommunicator +except ImportError: + raise ImportError("NCCL in Ray requires Cupy being available!") + +from ray.util.collective.types import ReduceOp, torch_available + +NCCL_REDUCE_OP_MAP = { + ReduceOp.SUM: nccl.NCCL_SUM, + ReduceOp.PRODUCT: nccl.NCCL_PROD, + ReduceOp.MIN: nccl.NCCL_MIN, + ReduceOp.MAX: nccl.NCCL_MAX, +} + +# cupy types are the same with numpy types +NUMPY_NCCL_DTYPE_MAP = { + numpy.uint8: nccl.NCCL_UINT8, + numpy.float16: nccl.NCCL_FLOAT16, + numpy.float32: nccl.NCCL_FLOAT32, + numpy.float64: nccl.NCCL_FLOAT64, +} + +if torch_available(): + import torch + TORCH_NCCL_DTYPE_MAP = { + torch.uint8: nccl.NCCL_UINT8, + torch.float16: nccl.NCCL_FLOAT16, + torch.float32: nccl.NCCL_FLOAT32, + torch.float64: nccl.NCCL_FLOAT64, + } + + +def get_nccl_build_version(): + return get_build_version() + + +def get_nccl_runtime_version(): + return get_version() + + +def get_nccl_unique_id(): + return nccl.get_unique_id() + + +def create_nccl_communicator(world_size, nccl_unique_id, rank): + """ + Create an NCCL communicator using NCCL APIs. + + Args: + world_size (int): the number of processes of this communcator group. + nccl_unique_id (str): the NCCLUniqueID for this group. + rank (int): the rank of this process. + Returns: + comm (nccl.ncclComm_t): an NCCL communicator. + """ + # TODO(Hao): make this inside the NCCLComm class, + # and implement the abort method. Make it RAII. + comm = NcclCommunicator(world_size, nccl_unique_id, rank) + return comm + + +def get_nccl_reduce_op(reduce_op): + """ + Map the reduce op to NCCL reduce op type. + + Args: + reduce_op (ReduceOp): ReduceOp Enum (SUM/PRODUCT/MIN/MAX). + Returns: + (nccl.ncclRedOp_t): the mapped NCCL reduce op. + """ + if reduce_op not in NCCL_REDUCE_OP_MAP: + raise RuntimeError( + "NCCL does not support reduce op: '{}'".format(reduce_op)) + return NCCL_REDUCE_OP_MAP[reduce_op] + + +def get_nccl_tensor_dtype(tensor): + """Return the corresponded NCCL dtype given a tensor.""" + if isinstance(tensor, cupy.ndarray): + return NUMPY_NCCL_DTYPE_MAP[tensor.dtype.type] + if torch_available(): + if isinstance(tensor, torch.Tensor): + return TORCH_NCCL_DTYPE_MAP[tensor.dtype] + raise ValueError("Unsupported tensor type. " + "Got: {}.".format(type(tensor))) + + +def get_tensor_ptr(tensor): + """Return the pointer to the underlying memory storage of a tensor.""" + if isinstance(tensor, cupy.ndarray): + return tensor.data.ptr + if isinstance(tensor, numpy.ndarray): + return tensor.data + if torch_available(): + if isinstance(tensor, torch.Tensor): + if not tensor.is_cuda: + raise RuntimeError("torch tensor must be on gpu.") + return tensor.data_ptr() + raise ValueError("Unsupported tensor type. " + "Got: {}.".format(type(tensor))) + + +def get_tensor_n_elements(tensor): + """Return the number of elements in a tensor.""" + if isinstance(tensor, cupy.ndarray) or isinstance(tensor, numpy.ndarray): + return tensor.size + if torch_available(): + if isinstance(tensor, torch.Tensor): + return torch.numel(tensor) + raise ValueError("Unsupported tensor type. " + "Got: {}.".format(type(tensor))) diff --git a/python/ray/util/collective/const.py b/python/ray/util/collective/const.py new file mode 100644 index 000000000..6eded9c51 --- /dev/null +++ b/python/ray/util/collective/const.py @@ -0,0 +1,21 @@ +""" +Constants. + +Contains constants used to setup collective groups. +""" +import hashlib + + +def get_nccl_store_name(group_name): + """ + Generate the unique name for the NCCLUniqueID store (named actor). + + Args: + group_name (str): unique user name for the store. + Return: + str: MD5-hexlified name for the store. + """ + if not group_name: + raise ValueError("group_name is None.") + hexlified_name = hashlib.md5(group_name.encode()).hexdigest() + return hexlified_name diff --git a/python/ray/util/collective/examples/nccl_allreduce_example.py b/python/ray/util/collective/examples/nccl_allreduce_example.py new file mode 100644 index 000000000..7010d6924 --- /dev/null +++ b/python/ray/util/collective/examples/nccl_allreduce_example.py @@ -0,0 +1,42 @@ +import ray +import cupy as cp + +import ray.util.collective as collective + + +@ray.remote(num_gpus=1) +class Worker: + def __init__(self): + self.send = cp.ones((4, ), dtype=cp.float32) + self.recv = cp.zeros((4, ), dtype=cp.float32) + + def setup(self, world_size, rank): + collective.init_collective_group("nccl", world_size, rank, "default") + return True + + def compute(self): + collective.allreduce(self.send, "default") + print(self.send) + return self.send + + def destroy(self): + collective.destroy_group() + + +if __name__ == "__main__": + + send = cp.ones((4, ), dtype=cp.float32) + + ray.init(num_gpus=2) + + num_workers = 2 + workers = [] + init_rets = [] + for i in range(num_workers): + w = Worker.remote() + workers.append(w) + init_rets.append(w.setup.remote(num_workers, i)) + _ = ray.get(init_rets) + results = ray.get([w.compute.remote() for w in workers]) + # print(results) + ray.shutdown() diff --git a/python/ray/util/collective/requirements.txt b/python/ray/util/collective/requirements.txt new file mode 100644 index 000000000..ce5057b22 --- /dev/null +++ b/python/ray/util/collective/requirements.txt @@ -0,0 +1 @@ +cupy-cuda100 \ No newline at end of file diff --git a/python/ray/util/collective/tests/__init__.py b/python/ray/util/collective/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/ray/util/collective/tests/conftest.py b/python/ray/util/collective/tests/conftest.py new file mode 100644 index 000000000..b84a01742 --- /dev/null +++ b/python/ray/util/collective/tests/conftest.py @@ -0,0 +1,37 @@ +"""Some fixtures for collective tests.""" +import pytest + +import ray +from ray.util.collective.const import get_nccl_store_name + + +def clean_up(): + group_names = ["default", "test", "123?34!", "default2", "random"] + group_names.extend([str(i) for i in range(10)]) + for group_name in group_names: + try: + store_name = get_nccl_store_name(group_name) + actor = ray.get_actor(store_name) + except ValueError: + actor = None + if actor: + ray.kill(actor) + + +@pytest.fixture +def ray_start_single_node_2_gpus(): + # Please start this fixture in a cluster with 2 GPUs. + address_info = ray.init(num_gpus=2) + yield address_info + ray.shutdown() + + +# Hao: this fixture is a bit tricky. +# I use a bash script to start a ray cluster on +# my own on-premise cluster before run this fixture. +@pytest.fixture +def ray_start_distributed_2_nodes_4_gpus(): + ray.init("auto") + yield + clean_up() + ray.shutdown() diff --git a/python/ray/util/collective/tests/test_collective_2_nodes_4_gpus.py b/python/ray/util/collective/tests/test_collective_2_nodes_4_gpus.py new file mode 100644 index 000000000..c35e48b9a --- /dev/null +++ b/python/ray/util/collective/tests/test_collective_2_nodes_4_gpus.py @@ -0,0 +1,276 @@ +"""Test the collective group APIs.""" +from random import shuffle +import pytest +import ray +from ray.util.collective.types import ReduceOp + +import cupy as cp +import torch + +from .util import Worker + + +def get_actors_group(num_workers=2, group_name="default", backend="nccl"): + actors = [Worker.remote() for i in range(num_workers)] + world_size = num_workers + init_results = ray.get([ + actor.init_group.remote(world_size, i, backend, group_name) + for i, actor in enumerate(actors) + ]) + return actors, init_results + + +@pytest.mark.parametrize("world_size", [2, 3, 4]) +@pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) +def test_init_two_actors(ray_start_distributed_2_nodes_4_gpus, world_size, + group_name): + actors, results = get_actors_group(world_size, group_name) + for i in range(world_size): + assert (results[i]) + + +@pytest.mark.parametrize("world_size", [2, 3, 4]) +def test_init_multiple_groups(ray_start_distributed_2_nodes_4_gpus, + world_size): + num_groups = 1 + actors = [Worker.remote() for _ in range(world_size)] + for i in range(num_groups): + group_name = str(i) + init_results = ray.get([ + actor.init_group.remote(world_size, i, group_name=group_name) + for i, actor in enumerate(actors) + ]) + for j in range(world_size): + assert init_results[j] + + +@pytest.mark.parametrize("world_size", [2, 3, 4]) +def test_get_rank(ray_start_distributed_2_nodes_4_gpus, world_size): + actors, _ = get_actors_group(world_size) + actor0_rank = ray.get(actors[0].report_rank.remote()) + assert actor0_rank == 0 + actor1_rank = ray.get(actors[1].report_rank.remote()) + assert actor1_rank == 1 + + # create a second group with a different name, and different + # orders of ranks. + new_group_name = "default2" + ranks = list(range(world_size)) + shuffle(ranks) + _ = ray.get([ + actor.init_group.remote( + world_size, ranks[i], group_name=new_group_name) + for i, actor in enumerate(actors) + ]) + actor0_rank = ray.get(actors[0].report_rank.remote(new_group_name)) + assert actor0_rank == ranks[0] + actor1_rank = ray.get(actors[1].report_rank.remote(new_group_name)) + assert actor1_rank == ranks[1] + + +@pytest.mark.parametrize("world_size", [2, 3, 4]) +def test_get_world_size(ray_start_distributed_2_nodes_4_gpus, world_size): + actors, _ = get_actors_group(world_size) + actor0_world_size = ray.get(actors[0].report_world_size.remote()) + actor1_world_size = ray.get(actors[1].report_world_size.remote()) + assert actor0_world_size == actor1_world_size == world_size + + +def test_availability(ray_start_distributed_2_nodes_4_gpus): + world_size = 4 + actors, _ = get_actors_group(world_size) + actor0_nccl_availability = ray.get( + actors[0].report_nccl_availability.remote()) + assert actor0_nccl_availability + actor0_mpi_availability = ray.get( + actors[0].report_mpi_availability.remote()) + assert not actor0_mpi_availability + + +def test_is_group_initialized(ray_start_distributed_2_nodes_4_gpus): + world_size = 4 + actors, _ = get_actors_group(world_size) + # check group is_init + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor0_is_init + actor0_is_init = ray.get( + actors[0].report_is_group_initialized.remote("random")) + assert not actor0_is_init + actor0_is_init = ray.get( + actors[0].report_is_group_initialized.remote("123")) + assert not actor0_is_init + actor1_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor1_is_init + actor1_is_init = ray.get( + actors[0].report_is_group_initialized.remote("456")) + assert not actor1_is_init + + +def test_destroy_group(ray_start_distributed_2_nodes_4_gpus): + world_size = 4 + actors, _ = get_actors_group(world_size) + # Now destroy the group at actor0 + ray.wait([actors[0].destroy_group.remote()]) + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert not actor0_is_init + + # should go well as the group `random` does not exist at all + ray.wait([actors[0].destroy_group.remote("random")]) + + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert actor1_is_init + ray.wait([actors[1].destroy_group.remote("random")]) + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert actor1_is_init + ray.wait([actors[1].destroy_group.remote("default")]) + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert not actor1_is_init + for i in [2, 3]: + ray.wait([actors[i].destroy_group.remote("default")]) + + # Now reconstruct the group using the same name + init_results = ray.get([ + actor.init_group.remote(world_size, i) + for i, actor in enumerate(actors) + ]) + for i in range(world_size): + assert init_results[i] + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor0_is_init + actor1_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor1_is_init + + +@pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) +@pytest.mark.parametrize("world_size", [2, 3, 4]) +def test_allreduce_different_name(ray_start_distributed_2_nodes_4_gpus, + group_name, world_size): + actors, _ = get_actors_group(num_workers=world_size, group_name=group_name) + results = ray.get([a.do_work.remote(group_name) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + + +@pytest.mark.parametrize("array_size", [2, 2**5, 2**10, 2**15, 2**20]) +def test_allreduce_different_array_size(ray_start_distributed_2_nodes_4_gpus, + array_size): + world_size = 4 + actors, _ = get_actors_group(world_size) + ray.wait([ + a.set_buffer.remote(cp.ones(array_size, dtype=cp.float32)) + for a in actors + ]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones( + (array_size, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones( + (array_size, ), dtype=cp.float32) * world_size).all() + + +def test_allreduce_destroy(ray_start_distributed_2_nodes_4_gpus, + backend="nccl", + group_name="default"): + world_size = 4 + actors, _ = get_actors_group(world_size) + + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + + # destroy the group and try do work, should fail + ray.wait([a.destroy_group.remote() for a in actors]) + with pytest.raises(RuntimeError): + results = ray.get([a.do_work.remote() for a in actors]) + + # reinit the same group and all reduce + ray.get([ + actor.init_group.remote(world_size, i, backend, group_name) + for i, actor in enumerate(actors) + ]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones( + (10, ), dtype=cp.float32) * world_size * world_size).all() + assert (results[1] == cp.ones( + (10, ), dtype=cp.float32) * world_size * world_size).all() + + +def test_allreduce_multiple_group(ray_start_distributed_2_nodes_4_gpus, + backend="nccl", + num_groups=5): + world_size = 4 + actors, _ = get_actors_group(world_size) + for group_name in range(1, num_groups): + ray.get([ + actor.init_group.remote(world_size, i, backend, str(group_name)) + for i, actor in enumerate(actors) + ]) + for i in range(num_groups): + group_name = "default" if i == 0 else str(i) + results = ray.get([a.do_work.remote(group_name) for a in actors]) + assert (results[0] == cp.ones( + (10, ), dtype=cp.float32) * (world_size**(i + 1))).all() + + +def test_allreduce_different_op(ray_start_distributed_2_nodes_4_gpus): + world_size = 4 + actors, _ = get_actors_group(world_size) + + # check product + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.PRODUCT) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 120).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 120).all() + + # check min + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.MIN) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 2).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 2).all() + + # check max + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.MAX) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 5).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 5).all() + + +@pytest.mark.parametrize("dtype", + [cp.uint8, cp.float16, cp.float32, cp.float64]) +def test_allreduce_different_dtype(ray_start_distributed_2_nodes_4_gpus, + dtype): + world_size = 4 + actors, _ = get_actors_group(world_size) + ray.wait([a.set_buffer.remote(cp.ones(10, dtype=dtype)) for a in actors]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=dtype) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=dtype) * world_size).all() + + +def test_allreduce_torch_cupy(ray_start_distributed_2_nodes_4_gpus): + # import torch + world_size = 4 + actors, _ = get_actors_group(world_size) + ray.wait([actors[1].set_buffer.remote(torch.ones(10, ).cuda())]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, )) * world_size).all() + + ray.wait([actors[0].set_buffer.remote(torch.ones(10, ))]) + ray.wait([actors[1].set_buffer.remote(cp.ones(10, ))]) + with pytest.raises(RuntimeError): + results = ray.get([a.do_work.remote() for a in actors]) + + +if __name__ == "__main__": + import pytest + import sys + + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/test_collective_single_node_2_gpus.py b/python/ray/util/collective/tests/test_collective_single_node_2_gpus.py new file mode 100644 index 000000000..267375e29 --- /dev/null +++ b/python/ray/util/collective/tests/test_collective_single_node_2_gpus.py @@ -0,0 +1,267 @@ +"""Test the collective group APIs.""" +import pytest +import ray +from ray.util.collective.types import ReduceOp + +import cupy as cp +import torch + +from .util import Worker + + +def get_actors_group(num_workers=2, group_name="default", backend="nccl"): + actors = [Worker.remote() for _ in range(num_workers)] + world_size = num_workers + init_results = ray.get([ + actor.init_group.remote(world_size, i, backend, group_name) + for i, actor in enumerate(actors) + ]) + return actors, init_results + + +@pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) +def test_init_two_actors(ray_start_single_node_2_gpus, group_name): + world_size = 2 + actors, results = get_actors_group(world_size, group_name) + for i in range(world_size): + assert (results[i]) + + +def test_init_multiple_groups(ray_start_single_node_2_gpus): + world_size = 2 + num_groups = 10 + actors = [Worker.remote() for i in range(world_size)] + for i in range(num_groups): + group_name = str(i) + init_results = ray.get([ + actor.init_group.remote(world_size, i, group_name=group_name) + for i, actor in enumerate(actors) + ]) + for j in range(world_size): + assert init_results[j] + + +def test_get_rank(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + actor0_rank = ray.get(actors[0].report_rank.remote()) + assert actor0_rank == 0 + actor1_rank = ray.get(actors[1].report_rank.remote()) + assert actor1_rank == 1 + + # create a second group with a different name, + # and different order of ranks. + new_group_name = "default2" + _ = ray.get([ + actor.init_group.remote( + world_size, world_size - 1 - i, group_name=new_group_name) + for i, actor in enumerate(actors) + ]) + actor0_rank = ray.get(actors[0].report_rank.remote(new_group_name)) + assert actor0_rank == 1 + actor1_rank = ray.get(actors[1].report_rank.remote(new_group_name)) + assert actor1_rank == 0 + + +def test_get_world_size(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + actor0_world_size = ray.get(actors[0].report_world_size.remote()) + actor1_world_size = ray.get(actors[1].report_world_size.remote()) + assert actor0_world_size == actor1_world_size == world_size + + +def test_availability(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + actor0_nccl_availability = ray.get( + actors[0].report_nccl_availability.remote()) + assert actor0_nccl_availability + actor0_mpi_availability = ray.get( + actors[0].report_mpi_availability.remote()) + assert not actor0_mpi_availability + + +def test_is_group_initialized(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + # check group is_init + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor0_is_init + actor0_is_init = ray.get( + actors[0].report_is_group_initialized.remote("random")) + assert not actor0_is_init + actor0_is_init = ray.get( + actors[0].report_is_group_initialized.remote("123")) + assert not actor0_is_init + actor1_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor1_is_init + actor1_is_init = ray.get( + actors[0].report_is_group_initialized.remote("456")) + assert not actor1_is_init + + +def test_destroy_group(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + # Now destroy the group at actor0 + ray.wait([actors[0].destroy_group.remote()]) + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert not actor0_is_init + + # should go well as the group `random` does not exist at all + ray.wait([actors[0].destroy_group.remote("random")]) + + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert actor1_is_init + ray.wait([actors[1].destroy_group.remote("random")]) + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert actor1_is_init + ray.wait([actors[1].destroy_group.remote("default")]) + actor1_is_init = ray.get(actors[1].report_is_group_initialized.remote()) + assert not actor1_is_init + + # Now reconstruct the group using the same name + init_results = ray.get([ + actor.init_group.remote(world_size, i) + for i, actor in enumerate(actors) + ]) + for i in range(world_size): + assert init_results[i] + actor0_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor0_is_init + actor1_is_init = ray.get(actors[0].report_is_group_initialized.remote()) + assert actor1_is_init + + +@pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) +# @pytest.mark.parametrize("group_name", ['123?34!']) +def test_allreduce_different_name(ray_start_single_node_2_gpus, group_name): + world_size = 2 + actors, _ = get_actors_group(num_workers=world_size, group_name=group_name) + results = ray.get([a.do_work.remote(group_name) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + + +@pytest.mark.parametrize("array_size", [2, 2**5, 2**10, 2**15, 2**20]) +def test_allreduce_different_array_size(ray_start_single_node_2_gpus, + array_size): + world_size = 2 + actors, _ = get_actors_group(world_size) + ray.wait([ + a.set_buffer.remote(cp.ones(array_size, dtype=cp.float32)) + for a in actors + ]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones( + (array_size, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones( + (array_size, ), dtype=cp.float32) * world_size).all() + + +def test_allreduce_destroy(ray_start_single_node_2_gpus, + backend="nccl", + group_name="default"): + world_size = 2 + actors, _ = get_actors_group(world_size) + + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * world_size).all() + + # destroy the group and try do work, should fail + ray.wait([a.destroy_group.remote() for a in actors]) + with pytest.raises(RuntimeError): + results = ray.get([a.do_work.remote() for a in actors]) + + # reinit the same group and all reduce + ray.get([ + actor.init_group.remote(world_size, i, backend, group_name) + for i, actor in enumerate(actors) + ]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones( + (10, ), dtype=cp.float32) * world_size * 2).all() + assert (results[1] == cp.ones( + (10, ), dtype=cp.float32) * world_size * 2).all() + + +def test_allreduce_multiple_group(ray_start_single_node_2_gpus, + backend="nccl", + num_groups=5): + world_size = 2 + actors, _ = get_actors_group(world_size) + for group_name in range(1, num_groups): + ray.get([ + actor.init_group.remote(world_size, i, backend, str(group_name)) + for i, actor in enumerate(actors) + ]) + for i in range(num_groups): + group_name = "default" if i == 0 else str(i) + results = ray.get([a.do_work.remote(group_name) for a in actors]) + assert (results[0] == cp.ones( + (10, ), dtype=cp.float32) * (world_size**(i + 1))).all() + + +def test_allreduce_different_op(ray_start_single_node_2_gpus): + world_size = 2 + actors, _ = get_actors_group(world_size) + + # check product + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.PRODUCT) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 6).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 6).all() + + # check min + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.MIN) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 2).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 2).all() + + # check max + ray.wait([ + a.set_buffer.remote(cp.ones(10, dtype=cp.float32) * (i + 2)) + for i, a in enumerate(actors) + ]) + results = ray.get([a.do_work.remote(op=ReduceOp.MAX) for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=cp.float32) * 3).all() + assert (results[1] == cp.ones((10, ), dtype=cp.float32) * 3).all() + + +@pytest.mark.parametrize("dtype", + [cp.uint8, cp.float16, cp.float32, cp.float64]) +def test_allreduce_different_dtype(ray_start_single_node_2_gpus, dtype): + world_size = 2 + actors, _ = get_actors_group(world_size) + ray.wait([a.set_buffer.remote(cp.ones(10, dtype=dtype)) for a in actors]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, ), dtype=dtype) * world_size).all() + assert (results[1] == cp.ones((10, ), dtype=dtype) * world_size).all() + + +def test_allreduce_torch_cupy(ray_start_single_node_2_gpus): + # import torch + world_size = 2 + actors, _ = get_actors_group(world_size) + ray.wait([actors[1].set_buffer.remote(torch.ones(10, ).cuda())]) + results = ray.get([a.do_work.remote() for a in actors]) + assert (results[0] == cp.ones((10, )) * world_size).all() + + ray.wait([actors[0].set_buffer.remote(torch.ones(10, ))]) + ray.wait([actors[1].set_buffer.remote(cp.ones(10, ))]) + with pytest.raises(RuntimeError): + results = ray.get([a.do_work.remote() for a in actors]) + + +if __name__ == "__main__": + import pytest + import sys + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/util.py b/python/ray/util/collective/tests/util.py new file mode 100644 index 000000000..d59294d3f --- /dev/null +++ b/python/ray/util/collective/tests/util.py @@ -0,0 +1,51 @@ +import cupy as cp + +import ray +import ray.util.collective as col +from ray.util.collective.types import Backend, ReduceOp + + +@ray.remote(num_gpus=1) +class Worker: + def __init__(self): + self.buffer = cp.ones((10, ), dtype=cp.float32) + + def init_group(self, + world_size, + rank, + backend=Backend.NCCL, + group_name="default"): + col.init_collective_group(world_size, rank, backend, group_name) + return True + + def set_buffer(self, data): + self.buffer = data + return self.buffer + + def do_work(self, group_name="default", op=ReduceOp.SUM): + col.allreduce(self.buffer, group_name, op) + return self.buffer + + def destroy_group(self, group_name="default"): + col.destroy_collective_group(group_name) + return True + + def report_rank(self, group_name="default"): + rank = col.get_rank(group_name) + return rank + + def report_world_size(self, group_name="default"): + ws = col.get_world_size(group_name) + return ws + + def report_nccl_availability(self): + avail = col.nccl_available() + return avail + + def report_mpi_availability(self): + avail = col.mpi_available() + return avail + + def report_is_group_initialized(self, group_name="default"): + is_init = col.is_group_initialized(group_name) + return is_init diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py new file mode 100644 index 000000000..ef037373a --- /dev/null +++ b/python/ray/util/collective/types.py @@ -0,0 +1,64 @@ +"""Types conversion between different backends.""" +from enum import Enum +from dataclasses import dataclass +from datetime import timedelta + +_NUMPY_AVAILABLE = True +_TORCH_AVAILABLE = True +_CUPY_AVAILABLE = True + +try: + import torch as th # noqa: F401 +except ImportError: + _TORCH_AVAILABLE = False + +try: + import cupy as cp # noqa: F401 +except ImportError: + _CUPY_AVAILABLE = False + + +def cupy_available(): + return _CUPY_AVAILABLE + + +def torch_available(): + return _TORCH_AVAILABLE + + +class Backend(object): + """A class to represent different backends.""" + NCCL = "nccl" + MPI = "mpi" + UNRECOGNIZED = "unrecognized" + + def __new__(cls, name: str): + backend = getattr(Backend, name.upper(), Backend.UNRECOGNIZED) + if backend == Backend.UNRECOGNIZED: + raise ValueError("Unrecognized backend: '{}'. " + "Only NCCL is supported".format(name)) + if backend == Backend.MPI: + raise NotImplementedError() + return backend + + +# TODO(Hao): extend this to support more MPI types +class ReduceOp(Enum): + SUM = 0 + PRODUCT = 1 + MIN = 2 + MAX = 3 + + +unset_timeout = timedelta(milliseconds=-1) + + +@dataclass +class AllReduceOptions: + reduceOp = ReduceOp.SUM + timeout = unset_timeout + + +@dataclass +class BarrierOptions: + timeout = unset_timeout diff --git a/python/ray/util/collective/util.py b/python/ray/util/collective/util.py new file mode 100644 index 000000000..e591e9b93 --- /dev/null +++ b/python/ray/util/collective/util.py @@ -0,0 +1,42 @@ +"""Some utility class for Collectives.""" +import ray +import logging + +logger = logging.getLogger(__name__) + + +@ray.remote +class NCCLUniqueIDStore: + """NCCLUniqueID Store as a named actor class. + + Args: + name (str): the unique name for this named actor. + + Attributes: + name (str): the unique name for this named actor. + nccl_id (str): the NCCLUniqueID held in this store. + """ + + def __init__(self, name): + self.name = name + self.nccl_id = None + + def set_id(self, uid): + """ + Initialize the NCCL unique ID for this store. + + Args: + uid (str): the unique ID generated via the NCCL get_unique_id API. + + Returns: + None + """ + self.nccl_id = uid + return self.nccl_id + + def get_id(self): + """Get the NCCL unique ID held in this store.""" + if not self.nccl_id: + logger.warning("The NCCL ID has not been " + "set yet for store {}.".format(self.name)) + return self.nccl_id diff --git a/python/ray/util/placement_group.py b/python/ray/util/placement_group.py index 8a6b6c3d1..182dbb38c 100644 --- a/python/ray/util/placement_group.py +++ b/python/ray/util/placement_group.py @@ -83,6 +83,19 @@ class PlacementGroup: placement_group_bundle_index=bundle_index, resources=resources).remote(self) + def wait(self, timeout_seconds: int) -> bool: + """Wait for the placement group to be ready within the specified time. + Args: + timeout_seconds(str): Timeout in seconds. + Return: + True if the placement group is created. False otherwise. + """ + worker = ray.worker.global_worker + worker.check_connected() + + return worker.core_worker.wait_placement_group_ready( + self.id, timeout_seconds) + @property def bundle_specs(self) -> List[Dict]: """List[Dict]: Return bundles belonging to this placement group.""" diff --git a/python/ray/util/queue.py b/python/ray/util/queue.py index cc6f6bd05..59be761de 100644 --- a/python/ray/util/queue.py +++ b/python/ray/util/queue.py @@ -1,4 +1,6 @@ import asyncio +from typing import Optional, Any, List +from collections.abc import Iterable import ray @@ -12,44 +14,69 @@ class Full(Exception): class Queue: - """Queue implementation on Ray. + """A first-in, first-out queue implementation on Ray. + + The behavior and use cases are similar to those of the asyncio.Queue class. + + Features both sync and async put and get methods. Provides the option to + block until space is available when calling put on a full queue, + or to block until items are available when calling get on an empty queue. + + Optionally supports batched put and get operations to minimize + serialization overhead. Args: - maxsize (int): maximum size of the queue. If zero, size is unbounded. + maxsize (optional, int): maximum size of the queue. If zero, size is + unbounded. + + Examples: + >>> q = Queue() + >>> items = list(range(10)) + >>> for item in items: + >>> q.put(item) + >>> for item in items: + >>> assert item == q.get() """ - def __init__(self, maxsize=0): - self.actor = _QueueActor.remote(maxsize) + def __init__(self, maxsize: int = 0) -> None: + self.maxsize = maxsize + self.actor = _QueueActor.remote(self.maxsize) - def __len__(self): + def __len__(self) -> int: return self.size() - def size(self): + def size(self) -> int: """The size of the queue.""" return ray.get(self.actor.qsize.remote()) - def qsize(self): + def qsize(self) -> int: """The size of the queue.""" return self.size() - def empty(self): + def empty(self) -> bool: """Whether the queue is empty.""" return ray.get(self.actor.empty.remote()) - def full(self): + def full(self) -> bool: """Whether the queue is full.""" return ray.get(self.actor.full.remote()) - def put(self, item, block=True, timeout=None): + def put(self, + item: Any, + block: bool = True, + timeout: Optional[float] = None) -> None: """Adds an item to the queue. + If block is True and the queue is full, blocks until the queue is no + longer full or until timeout. + There is no guarantee of order if multiple producers put to the same full queue. Raises: - Full if the queue is full and blocking is False. - Full if the queue is full, blocking is True, and it timed out. - ValueError if timeout is negative. + Full: if the queue is full and blocking is False. + Full: if the queue is full, blocking is True, and it timed out. + ValueError: if timeout is negative. """ if not block: try: @@ -62,9 +89,40 @@ class Queue: else: ray.get(self.actor.put.remote(item, timeout)) - def get(self, block=True, timeout=None): + async def put_async(self, + item: Any, + block: bool = True, + timeout: Optional[float] = None) -> None: + """Adds an item to the queue. + + If block is True and the queue is full, + blocks until the queue is no longer full or until timeout. + + There is no guarantee of order if multiple producers put to the same + full queue. + + Raises: + Full: if the queue is full and blocking is False. + Full: if the queue is full, blocking is True, and it timed out. + ValueError: if timeout is negative. + """ + if not block: + try: + await self.actor.put_nowait.remote(item) + except asyncio.QueueFull: + raise Full + else: + if timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + await self.actor.put.remote(item, timeout) + + def get(self, block: bool = True, timeout: Optional[float] = None) -> Any: """Gets an item from the queue. + If block is True and the queue is empty, blocks until the queue is no + longer empty or until timeout. + There is no guarantee of order if multiple consumers get from the same empty queue. @@ -72,9 +130,9 @@ class Queue: The next item in the queue. Raises: - Empty if the queue is empty and blocking is False. - Empty if the queue is empty, blocking is True, and it timed out. - ValueError if timeout is negative. + Empty: if the queue is empty and blocking is False. + Empty: if the queue is empty, blocking is True, and it timed out. + ValueError: if timeout is negative. """ if not block: try: @@ -87,27 +145,79 @@ class Queue: else: return ray.get(self.actor.get.remote(timeout)) - def put_nowait(self, item): + async def get_async(self, + block: bool = True, + timeout: Optional[float] = None) -> Any: + """Gets an item from the queue. + + There is no guarantee of order if multiple consumers get from the + same empty queue. + + Returns: + The next item in the queue. + Raises: + Empty: if the queue is empty and blocking is False. + Empty: if the queue is empty, blocking is True, and it timed out. + ValueError: if timeout is negative. + """ + if not block: + try: + return await self.actor.get_nowait.remote() + except asyncio.QueueEmpty: + raise Empty + else: + if timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + else: + return await self.actor.get.remote(timeout) + + def put_nowait(self, item: Any) -> None: """Equivalent to put(item, block=False). Raises: - Full if the queue is full. + Full: if the queue is full. """ return self.put(item, block=False) - def get_nowait(self): + def put_nowait_batch(self, items: Iterable) -> None: + """Takes in a list of items and puts them into the queue in order. + + Raises: + Full: if the items will not fit in the queue + """ + if not isinstance(items, Iterable): + raise TypeError("Argument 'items' must be an Iterable") + + ray.get(self.actor.put_nowait_batch.remote(items)) + + def get_nowait(self) -> Any: """Equivalent to get(block=False). Raises: - Empty if the queue is empty. + Empty: if the queue is empty. """ return self.get(block=False) + def get_nowait_batch(self, num_items: int) -> List[Any]: + """Gets items from the queue and returns them in a + list in order. + + Raises: + Empty: if the queue does not contain the desired number of items + """ + if not isinstance(num_items, int): + raise TypeError("Argument 'num_items' must be an int") + if num_items < 0: + raise ValueError("'num_items' must be nonnegative") + + return ray.get(self.actor.get_nowait_batch.remote(num_items)) + @ray.remote class _QueueActor: def __init__(self, maxsize): - self.queue = asyncio.Queue(maxsize) + self.maxsize = maxsize + self.queue = asyncio.Queue(self.maxsize) def qsize(self): return self.queue.qsize() @@ -133,5 +243,19 @@ class _QueueActor: def put_nowait(self, item): self.queue.put_nowait(item) + def put_nowait_batch(self, items): + # If maxsize is 0, queue is unbounded, so no need to check size. + if self.maxsize > 0 and len(items) + self.qsize() > self.maxsize: + raise Full(f"Cannot add {len(items)} items to queue of size " + f"{self.qsize()} and maxsize {self.maxsize}.") + for item in items: + self.queue.put_nowait(item) + def get_nowait(self): return self.queue.get_nowait() + + def get_nowait_batch(self, num_items): + if num_items > self.qsize(): + raise Empty(f"Cannot get {num_items} items from queue of size " + f"{self.qsize()}.") + return [self.queue.get_nowait() for _ in range(num_items)] diff --git a/python/ray/util/rpdb.py b/python/ray/util/rpdb.py index 3ffd3626f..33d430ea0 100644 --- a/python/ray/util/rpdb.py +++ b/python/ray/util/rpdb.py @@ -15,6 +15,7 @@ from pdb import Pdb import setproctitle import traceback +import ray from ray.experimental.internal_kv import _internal_kv_del, _internal_kv_put PY3 = sys.version_info[0] == 3 @@ -70,7 +71,13 @@ class RemotePdb(Pdb): """ active_instance = None - def __init__(self, host, port, patch_stdstreams=False, quiet=False): + def __init__(self, + breakpoint_uuid, + host, + port, + patch_stdstreams=False, + quiet=False): + self._breakpoint_uuid = breakpoint_uuid self._quiet = quiet self._patch_stdstreams = patch_stdstreams self._listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -138,8 +145,35 @@ class RemotePdb(Pdb): if exc.errno != errno.ECONNRESET: raise + def do_remote(self, arg): + """remote + Skip into the next remote call. + """ + # Tell the next task to drop into the debugger. + ray.worker.global_worker.debugger_breakpoint = self._breakpoint_uuid + # Tell the debug loop to connect to the next task. + _internal_kv_put("RAY_PDB_CONTINUE_{}".format(self._breakpoint_uuid), + "") + self.__restore() + self.handle.connection.close() + return Pdb.do_continue(self, arg) -def connect_ray_pdb(host=None, port=None, patch_stdstreams=False, quiet=None): + def do_get(self, arg): + """get + Skip to where the current task returns to. + """ + ray.worker.global_worker.debugger_get_breakpoint = ( + self._breakpoint_uuid) + self.__restore() + self.handle.connection.close() + return Pdb.do_continue(self, arg) + + +def connect_ray_pdb(host=None, + port=None, + patch_stdstreams=False, + quiet=None, + breakpoint_uuid=None): """ Opens a remote PDB on first available port. """ @@ -149,8 +183,14 @@ def connect_ray_pdb(host=None, port=None, patch_stdstreams=False, quiet=None): port = int(os.environ.get("REMOTE_PDB_PORT", "0")) if quiet is None: quiet = bool(os.environ.get("REMOTE_PDB_QUIET", "")) + if not breakpoint_uuid: + breakpoint_uuid = uuid.uuid4().hex rdb = RemotePdb( - host=host, port=port, patch_stdstreams=patch_stdstreams, quiet=quiet) + breakpoint_uuid=breakpoint_uuid, + host=host, + port=port, + patch_stdstreams=patch_stdstreams, + quiet=quiet) sockname = rdb._listen_socket.getsockname() pdb_address = "{}:{}".format(sockname[0], sockname[1]) parentframeinfo = inspect.getouterframes(inspect.currentframe())[2] @@ -161,7 +201,6 @@ def connect_ray_pdb(host=None, port=None, patch_stdstreams=False, quiet=None): "lineno": parentframeinfo.lineno, "traceback": "\n".join(traceback.format_exception(*sys.exc_info())) } - breakpoint_uuid = uuid.uuid4() _internal_kv_put( "RAY_PDB_{}".format(breakpoint_uuid), json.dumps(data), overwrite=True) rdb.listen() @@ -170,14 +209,19 @@ def connect_ray_pdb(host=None, port=None, patch_stdstreams=False, quiet=None): return rdb -def set_trace(): +def set_trace(breakpoint_uuid=None): """Interrupt the flow of the program and drop into the Ray debugger. Can be used within a Ray task or actor. """ - frame = sys._getframe().f_back - rdb = connect_ray_pdb(None, None, False, None) - rdb.set_trace(frame=frame) + # If there is an active debugger already, we do not want to + # start another one, so "set_trace" is just a no-op in that case. + if ray.worker.global_worker.debugger_breakpoint == b"": + frame = sys._getframe().f_back + rdb = connect_ray_pdb( + None, None, False, None, + breakpoint_uuid.decode() if breakpoint_uuid else None) + rdb.set_trace(frame=frame) def post_mortem(): diff --git a/python/ray/util/sgd/tests/test_torch_failure.py b/python/ray/util/sgd/tests/test_torch_failure.py index 97324e0a5..4f48daaec 100644 --- a/python/ray/util/sgd/tests/test_torch_failure.py +++ b/python/ray/util/sgd/tests/test_torch_failure.py @@ -93,14 +93,22 @@ def test_resize(ray_start_2_cpus, use_local): # noqa: F811 use_local=use_local, num_workers=2) - @ray.remote - def try_test(): - import time - time.sleep(100) + @ray.remote(num_cpus=1) + class DummyActor: + def get(self): + return 1 - try_test.remote() + dummy_handler = DummyActor.remote() trainer1.train(max_retries=1) assert trainer1.worker_group.num_workers == 1 + assert trainer1._num_failures == 1 + + ray.get(dummy_handler.get.remote()) + ray.kill(dummy_handler) + time.sleep(1) + # trigger scale up + trainer1.train() + assert trainer1.worker_group.num_workers == 2 trainer1.shutdown(force=True) @@ -132,6 +140,8 @@ def test_fail_twice(ray_start_2_cpus, use_local): # noqa: F811 # MAX RETRIES SHOULD BE ON BY DEFAULT trainer1.train() + assert trainer1._num_failures == 2 + assert trainer1.worker_group.num_workers == 2 trainer1.shutdown(force=True) diff --git a/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml b/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml index 392d91937..2f5b74750 100644 --- a/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml +++ b/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml @@ -48,7 +48,7 @@ worker_nodes: setup_commands: - conda install setuptools=45.1.0=py36_0 wrapt=1.11.2 --yes # workaround to fix wrapt error - - ray &> /dev/null || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray &> /dev/null || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl - pip install -U ray[tune] - pip install tensorflow-gpu==2.1.0 diff --git a/python/ray/util/sgd/torch/examples/dcgan.py b/python/ray/util/sgd/torch/examples/dcgan.py index 9bbf34f41..936272be2 100644 --- a/python/ray/util/sgd/torch/examples/dcgan.py +++ b/python/ray/util/sgd/torch/examples/dcgan.py @@ -182,8 +182,8 @@ class GANOperator(TrainingOperator): @override(TrainingOperator) def train_batch(self, batch, batch_info): """Trains on one batch of data from the data creator.""" - real_label = 1 - fake_label = 0 + real_label = 1.0 + fake_label = 0. discriminator, generator = self.models optimD, optimG = self.optimizers diff --git a/python/ray/util/sgd/torch/examples/example-sgd.yaml b/python/ray/util/sgd/torch/examples/example-sgd.yaml index 743ec2488..42fa7a4ad 100644 --- a/python/ray/util/sgd/torch/examples/example-sgd.yaml +++ b/python/ray/util/sgd/torch/examples/example-sgd.yaml @@ -43,7 +43,7 @@ worker_nodes: # MaxPrice: "9.0" setup_commands: - - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl - pip install -U ipdb ray[rllib] torch torchvision # Install apex. # - rm -rf apex || true diff --git a/python/ray/util/sgd/torch/examples/image_models/cluster.yaml b/python/ray/util/sgd/torch/examples/image_models/cluster.yaml index 14577161f..6d4e56228 100644 --- a/python/ray/util/sgd/torch/examples/image_models/cluster.yaml +++ b/python/ray/util/sgd/torch/examples/image_models/cluster.yaml @@ -62,7 +62,7 @@ worker_nodes: setup_commands: # This replaces the standard anaconda Ray installation - - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl # Uncomment this and the filemount to update the Ray installation with your local Ray code # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/ # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/ diff --git a/python/ray/util/sgd/torch/examples/segmentation/example.yaml b/python/ray/util/sgd/torch/examples/segmentation/example.yaml index c90c62a7d..eebe0f695 100644 --- a/python/ray/util/sgd/torch/examples/segmentation/example.yaml +++ b/python/ray/util/sgd/torch/examples/segmentation/example.yaml @@ -42,7 +42,7 @@ worker_nodes: setup_commands: # This replaces the standard anaconda Ray installation - - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl # Uncomment this and the filemount to update the Ray installation with your local Ray code # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/ # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/ diff --git a/python/ray/util/sgd/torch/examples/sgd-development.yaml b/python/ray/util/sgd/torch/examples/sgd-development.yaml index d3c089cc8..12863f99f 100644 --- a/python/ray/util/sgd/torch/examples/sgd-development.yaml +++ b/python/ray/util/sgd/torch/examples/sgd-development.yaml @@ -62,7 +62,7 @@ worker_nodes: setup_commands: # This replaces the standard anaconda Ray installation - - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl # Uncomment this and the filemount to update the Ray installation with your local Ray code # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/ # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/ diff --git a/python/ray/util/sgd/torch/examples/transformers/cluster.yaml b/python/ray/util/sgd/torch/examples/transformers/cluster.yaml index 2aeb7087d..39c012f16 100644 --- a/python/ray/util/sgd/torch/examples/transformers/cluster.yaml +++ b/python/ray/util/sgd/torch/examples/transformers/cluster.yaml @@ -47,7 +47,7 @@ worker_nodes: setup_commands: # This replaces the standard anaconda Ray installation - - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl + - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl - pip install -q tqdm # Installing this without -U to make sure we don't replace the existing Ray installation diff --git a/python/ray/util/sgd/torch/torch_trainer.py b/python/ray/util/sgd/torch/torch_trainer.py index 0602222c5..d17c23191 100644 --- a/python/ray/util/sgd/torch/torch_trainer.py +++ b/python/ray/util/sgd/torch/torch_trainer.py @@ -300,7 +300,7 @@ class TorchTrainer: wrap_ddp=self.wrap_ddp) worker_args = { - "max_workers": num_workers, + "max_workers": self.max_replicas, "params": params, "dist_params": dist_params, "initialization_hook": self.initialization_hook, diff --git a/python/ray/util/sgd/torch/worker_group.py b/python/ray/util/sgd/torch/worker_group.py index 9a42bf46b..f1a82082a 100644 --- a/python/ray/util/sgd/torch/worker_group.py +++ b/python/ray/util/sgd/torch/worker_group.py @@ -381,7 +381,7 @@ class RemoteWorkerGroup(WorkerGroupInterface): past_cooldown = (time.time() - self._last_resize) > RESIZE_COOLDOWN_S if past_cooldown and worker_gap: # Assume 1 resource is already reserved for local worker. - potential_remote_size = self._check_potential_remote_workers_size() + potential_remote_size = self.new_workers_size() return potential_remote_size > 0 return False @@ -514,7 +514,10 @@ class LocalWorkerGroup(WorkerGroupInterface): def reset(self): """Terminates models without giving up local resource reservation.""" - self.local_worker.shutdown(cleanup=False) + if not isinstance(self.local_worker, LocalDistributedRunner): + self.local_worker.shutdown() + else: + self.local_worker.shutdown(cleanup=False) self.remote_worker_group.reset() self.local_worker = None diff --git a/python/ray/util/xgboost/BUILD b/python/ray/util/xgboost/BUILD index 41e48cf15..ecff05963 100644 --- a/python/ray/util/xgboost/BUILD +++ b/python/ray/util/xgboost/BUILD @@ -2,25 +2,25 @@ # Tests from the python/ray/util/sgd/tests directory. # Please keep these sorted alphabetically. # -------------------------------------------------------------------- -py_test( - name = "simple_example", - size = "small", - srcs = ["simple_example.py"], - deps = [":xgb_lib"], - tags = ["exclusive"], -) +# py_test( +# name = "simple_example", +# size = "small", +# srcs = ["simple_example.py"], +# deps = [":xgb_lib"], +# tags = ["exclusive"], +# ) -py_test( - name = "simple_tune", - size="small", - srcs = ["simple_tune.py"], - deps = [":xgb_lib"], - tags = ["exlcusive"] -) +# py_test( +# name = "simple_tune", +# size="small", +# srcs = ["simple_tune.py"], +# deps = [":xgb_lib"], +# tags = ["exlcusive"] +# ) -# This is a dummy test dependency that causes the above tests to be -# re-run if any of these files changes. -py_library( - name = "xgb_lib", - srcs = glob(["**/*.py"], exclude=["tests/*.py"]), -) \ No newline at end of file +# # This is a dummy test dependency that causes the above tests to be +# # re-run if any of these files changes. +# py_library( +# name = "xgb_lib", +# srcs = glob(["**/*.py"], exclude=["tests/*.py"]), +# ) diff --git a/python/ray/worker.py b/python/ray/worker.py index d5093e360..cc231f7fa 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -102,6 +102,14 @@ class Worker: # Index of the current session. This number will # increment every time when `ray.shutdown` is called. self._session_index = 0 + # If this is set, the next .remote call should drop into the + # debugger, at the specified breakpoint ID. + self.debugger_breakpoint = b"" + # If this is set, ray.get calls invoked on the object ID returned + # by the worker should drop into the debugger at the specified + # breakpoint ID. + self.debugger_get_breakpoint = b"" + self._load_code_from_local = False @property def connected(self): @@ -115,7 +123,7 @@ class Worker: @property def load_code_from_local(self): self.check_connected() - return self.node.load_code_from_local + return self._load_code_from_local @property def current_job_id(self): @@ -215,6 +223,9 @@ class Worker: """ self.mode = mode + def set_load_code_from_local(self, load_code_from_local): + self._load_code_from_local = load_code_from_local + def put_object(self, value, object_ref=None, pin_object=True): """Put value in the local object store with object reference `object_ref`. @@ -280,6 +291,10 @@ class Worker: whose values should be retrieved. timeout (float): timeout (float): The maximum amount of time in seconds to wait before returning. + Returns: + list: List of deserialized objects + bytes: UUID of the debugger breakpoint we should drop + into or b"" if there is no breakpoint. """ # Make sure that the values are object refs. for object_ref in object_refs: @@ -291,7 +306,16 @@ class Worker: timeout_ms = int(timeout * 1000) if timeout else -1 data_metadata_pairs = self.core_worker.get_objects( object_refs, self.current_task_id, timeout_ms) - return self.deserialize_objects(data_metadata_pairs, object_refs) + debugger_breakpoint = b"" + for (data, metadata) in data_metadata_pairs: + if metadata: + metadata_fields = metadata.split(b",") + if len(metadata_fields) >= 2 and metadata_fields[1].startswith( + ray_constants.OBJECT_METADATA_DEBUG_PREFIX): + debugger_breakpoint = metadata_fields[1][len( + ray_constants.OBJECT_METADATA_DEBUG_PREFIX):] + return self.deserialize_objects(data_metadata_pairs, + object_refs), debugger_breakpoint def run_function_on_all_workers(self, function, run_on_other_drivers=False): @@ -469,9 +493,7 @@ def init( _memory=None, _redis_password=ray_constants.REDIS_DEFAULT_PASSWORD, _java_worker_options=None, - _code_search_path=None, _temp_dir=None, - _load_code_from_local=False, _lru_evict=False, _metrics_export_port=None, _system_config=None): @@ -559,10 +581,7 @@ def init( _temp_dir (str): If provided, specifies the root temporary directory for the Ray process. Defaults to an OS-specific conventional location, e.g., "/tmp/ray". - _load_code_from_local: Whether code should be loaded from a local - module or from the GCS. _java_worker_options: Overwrite the options to start Java workers. - _code_search_path (list): Java classpath or python import path. _lru_evict (bool): If True, when an object store is full, it will evict objects in LRU order to make more space and when under memory pressure, ray.ObjectLostError may be thrown. If False, then @@ -589,10 +608,12 @@ def init( import resource soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) if soft < hard: + # https://github.com/ray-project/ray/issues/12059 + soft = max(soft, min(hard, 65536)) logger.debug("Automatically increasing RLIMIT_NOFILE to max " "value of {}".format(hard)) try: - resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) except ValueError: logger.debug("Failed to raise limit.") soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) @@ -681,9 +702,7 @@ def init( redis_max_memory=_redis_max_memory, plasma_store_socket_name=None, temp_dir=_temp_dir, - load_code_from_local=_load_code_from_local, java_worker_options=_java_worker_options, - code_search_path=_code_search_path, start_initial_python_workers_for_first_job=True, _system_config=_system_config, lru_evict=_lru_evict, @@ -729,7 +748,6 @@ def init( redis_password=_redis_password, object_ref_seed=None, temp_dir=_temp_dir, - load_code_from_local=_load_code_from_local, _system_config=_system_config, lru_evict=_lru_evict, enable_object_reconstruction=_enable_object_reconstruction, @@ -833,6 +851,7 @@ def custom_excepthook(type, value, tb): worker_type = ray.gcs_utils.DRIVER worker_info = {"exception": error_message} + ray.state.state._check_connected() ray.state.state.add_worker(worker_id, worker_type, worker_info) # Call the normal excepthook. normal_excepthook(type, value, tb) @@ -1345,7 +1364,8 @@ def get(object_refs, *, timeout=None): global last_task_error_raise_time # TODO(ujvl): Consider how to allow user to retrieve the ready objects. - values = worker.get_objects(object_refs, timeout=timeout) + values, debugger_breakpoint = worker.get_objects( + object_refs, timeout=timeout) for i, value in enumerate(values): if isinstance(value, RayError): last_task_error_raise_time = time.time() @@ -1358,6 +1378,14 @@ def get(object_refs, *, timeout=None): if is_individual_id: values = values[0] + + if debugger_breakpoint != b"": + frame = sys._getframe().f_back + rdb = ray.util.pdb.connect_ray_pdb( + None, None, False, None, + debugger_breakpoint.decode() if debugger_breakpoint else None) + rdb.set_trace(frame=frame) + return values @@ -1507,11 +1535,12 @@ def kill(actor, *, no_restart=True): """Kill an actor forcefully. This will interrupt any running tasks on the actor, causing them to fail - immediately. Any atexit handlers installed in the actor will still be run. + immediately. ``atexit`` handlers installed in the actor will not be run. If you want to kill the actor but let pending tasks finish, you can call ``actor.__ray_terminate__.remote()`` instead to queue a - termination task. + termination task. Any ``atexit`` handlers installed in the actor *will* + be run in this case. If the actor is a detached actor, subsequent calls to get its handle via ray.get_actor will fail. diff --git a/python/ray/workers/default_worker.py b/python/ray/workers/default_worker.py index 5574b2e4c..d9f7837ff 100644 --- a/python/ray/workers/default_worker.py +++ b/python/ray/workers/default_worker.py @@ -11,8 +11,7 @@ import ray.node import ray.ray_constants as ray_constants import ray.utils from ray.parameter import RayParams -from ray.ray_logging import (StandardStreamInterceptor, - setup_and_get_worker_interceptor_logger) +from ray.ray_logging import get_worker_log_file_name, configure_log_file parser = argparse.ArgumentParser( description=("Parse addresses for the worker " @@ -145,11 +144,14 @@ if __name__ == "__main__": raylet_ip_address = args.node_ip_address code_search_path = args.code_search_path + load_code_from_local = False if code_search_path is not None: + load_code_from_local = True for p in code_search_path.split(":"): if os.path.isfile(p): p = os.path.dirname(p) sys.path.append(p) + ray.worker.global_worker.set_load_code_from_local(load_code_from_local) ray_params = RayParams( node_ip_address=args.node_ip_address, @@ -160,7 +162,6 @@ if __name__ == "__main__": plasma_store_socket_name=args.object_store_name, raylet_socket_name=args.raylet_name, temp_dir=args.temp_dir, - load_code_from_local=args.load_code_from_local, metrics_agent_port=args.metrics_agent_port, ) @@ -173,20 +174,10 @@ if __name__ == "__main__": ray.worker._global_node = node ray.worker.connect(node, mode=mode) - # Redirect stdout and stderr to the default worker interceptor logger. - # NOTE: We deprecated redirect_worker_output arg, - # so we don't need to handle here. - stdout_interceptor = StandardStreamInterceptor( - setup_and_get_worker_interceptor_logger(args, is_for_stdout=True), - intercept_stdout=True) - stderr_interceptor = StandardStreamInterceptor( - setup_and_get_worker_interceptor_logger(args, is_for_stdout=False), - intercept_stdout=False) - # Although the os level fd is duplicated already, we should overwrite - # the python level stdout/stderr object. - # Otherwise, buffers won't be flushed. - sys.stdout = stdout_interceptor - sys.stderr = stderr_interceptor + # Setup log file. + out_file, err_file = node.get_log_file_handles( + get_worker_log_file_name(args.worker_type)) + configure_log_file(out_file, err_file) if mode == ray.WORKER_MODE: ray.worker.global_worker.main_loop() diff --git a/python/requirements.txt b/python/requirements.txt index 19b01dfc1..28c387fde 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -38,6 +38,7 @@ tensorboardX uvicorn pydantic<1.7 dataclasses; python_version < '3.7' +starlette # Requirements for running tests blist; platform_system != "Windows" @@ -57,6 +58,7 @@ numba # higher version of llvmlite breaks windows llvmlite==0.34.0 openpyxl +pexpect Pillow; platform_system != "Windows" pygments pytest==5.4.3 diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt index 1c75eb888..d68d3b3d3 100644 --- a/python/requirements_tune.txt +++ b/python/requirements_tune.txt @@ -1,5 +1,4 @@ ax-platform -gpytorch==1.2 bayesian-optimization ConfigSpace==0.4.10 dragonfly-opt diff --git a/python/setup.py b/python/setup.py index 27d256f5c..8be61e9dd 100644 --- a/python/setup.py +++ b/python/setup.py @@ -95,10 +95,9 @@ ray_files += [ # also update the matching section of requirements.txt # in this directory extras = { - "debug": [], "serve": [ "uvicorn", "flask", "requests", "pydantic<1.7", - "dataclasses; python_version < '3.7'" + "dataclasses; python_version < '3.7'", "starlette" ], "tune": [ "dataclasses; python_version < '3.7'", "pandas", "tabulate", @@ -456,8 +455,10 @@ setuptools.setup( entry_points={ "console_scripts": [ "ray=ray.scripts.scripts:main", - "rllib=ray.rllib.scripts:cli [rllib]", "tune=ray.tune.scripts:cli", - "ray-operator=ray.operator:main" + "rllib=ray.rllib.scripts:cli [rllib]", + "tune=ray.tune.scripts:cli", + "ray-operator=ray.operator.operator:main", + "serve=ray.serve.scripts:cli", ] }, include_package_data=True, diff --git a/release/long_running_distributed_tests/cluster.yaml b/release/long_running_distributed_tests/cluster.yaml index a2ed252a4..d4605e2ed 100644 --- a/release/long_running_distributed_tests/cluster.yaml +++ b/release/long_running_distributed_tests/cluster.yaml @@ -1,64 +1,36 @@ -# This file is generated by `ray project create`. - -# A unique identifier for the head node and workers of this cluster. cluster_name: long-running-distributed-tests -# The minimum number of workers nodes to launch in addition to the head -# node. This number should be >= 0. min_workers: 3 -# The maximum number of workers nodes to launch in addition to the head -# node. This takes precedence over min_workers. min_workers defaults to 0. max_workers: 3 -# The autoscaler will scale up the cluster to this target fraction of resource -# usage. For example, if a cluster of 10 nodes is 100% busy and -# target_utilization is 0.8, it would resize the cluster to 13. This fraction -# can be decreased to increase the aggressiveness of upscaling. -# This value must be less than 1.0 for scaling to happen. target_utilization_fraction: 0.8 +idle_timeout_minutes: 15 -# If a node is idle for this many minutes, it will be removed. -idle_timeout_minutes: 5 +docker: + image: anyscale/ray-ml:latest-gpu + container_name: ray_container + pull_before_run: True -# Cloud-provider specific configuration. provider: type: aws region: us-west-2 availability_zone: us-west-2a cache_stopped_nodes: False -# How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu -# By default Ray creates a new private keypair, but you can also use your own. -# If you do so, make sure to also set "KeyName" in the head and worker node -# configurations below. -# ssh_private_key: /path/to/your/key.pem - -# Provider-specific config for the head node, e.g. instance type. By default -# Ray will auto-configure unspecified fields such as SubnetId and KeyName. -# For more documentation on available fields, see: -# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: InstanceType: g3.8xlarge - ImageId: ami-0888a3b5189309429 # DLAMI 7/1/19 - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 150 worker_nodes: InstanceType: g3.8xlarge - ImageId: ami-0888a3b5189309429 # DLAMI 7/1/19 - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 150 InstanceMarketOptions: MarketType: spot -setup_commands: [] +setup_commands: + - apt-get install -y libglib2.0-0 libcudnn7=7.6.5.32-1+cuda10.1 + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: diff --git a/release/long_running_distributed_tests/run.sh b/release/long_running_distributed_tests/run.sh index 386416a08..d0fa4a6c4 100755 --- a/release/long_running_distributed_tests/run.sh +++ b/release/long_running_distributed_tests/run.sh @@ -42,7 +42,7 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: $workload" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp37-cp37m-manylinux2014_x86_64.whl" conda uninstall -y terminado || true pip install -U pip diff --git a/release/long_running_distributed_tests/workloads/pytorch_pbt_failure.py b/release/long_running_distributed_tests/workloads/pytorch_pbt_failure.py index 0fa94cb44..2451fe4a2 100644 --- a/release/long_running_distributed_tests/workloads/pytorch_pbt_failure.py +++ b/release/long_running_distributed_tests/workloads/pytorch_pbt_failure.py @@ -13,7 +13,7 @@ from ray.tune import CLIReporter from ray.tune.schedulers import PopulationBasedTraining from ray.tune.utils.util import merge_dicts from ray.tune.utils.mock import FailureInjectorCallback -from ray.util.sgd.torch import TorchTrainer +from ray.util.sgd.torch import TorchTrainer, TrainingOperator from ray.util.sgd.torch.resnet import ResNet18 from ray.util.sgd.utils import BATCH_SIZE @@ -74,13 +74,17 @@ def optimizer_creator(model, config): momentum=config.get("momentum", 0.9)) -ray.init(address="auto" if not args.smoke_test else None, _log_to_driver=True) +ray.init(address="auto" if not args.smoke_test else None, log_to_driver=True) num_training_workers = 1 if args.smoke_test else 3 -TorchTrainable = TorchTrainer.as_trainable( + +CustomTrainingOperator = TrainingOperator.from_creators( model_creator=ResNet18, - data_creator=cifar_creator, optimizer_creator=optimizer_creator, - loss_creator=nn.CrossEntropyLoss, + data_creator=cifar_creator, + loss_creator=nn.CrossEntropyLoss) + +TorchTrainable = TorchTrainer.as_trainable( + training_operator_cls=CustomTrainingOperator, initialization_hook=initialization_hook, num_workers=num_training_workers, config={ diff --git a/release/long_running_tests/cluster.yaml b/release/long_running_tests/cluster.yaml index 074d445ba..e51cd010b 100644 --- a/release/long_running_tests/cluster.yaml +++ b/release/long_running_tests/cluster.yaml @@ -1,48 +1,17 @@ -cluster_name: default -min_workers: 0 -max_workers: 0 -target_utilization_fraction: 0.8 -idle_timeout_minutes: 5 +cluster_name: ray-release-long-running-tests + +docker: + image: anyscale/ray:latest + container_name: ray_container + pull_before_run: False -# Cloud-provider specific configuration. provider: type: aws region: us-west-2 - availability_zone: us-west-2a + availability_zone: us-west-2a, us-west-2b, us-west-2c + auth: ssh_user: ubuntu head_node: - InstanceType: m5.2xlarge - ImageId: ami-0888a3b5189309429 # DLAMI 7/1/19 - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 150 - -worker_nodes: - InstanceType: m5.large - ImageId: ami-0888a3b5189309429 # DLAMI 7/1/19 - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 150 - - # Run workers on spot by default. Comment this out to use on-demand. - InstanceMarketOptions: - MarketType: spot - -# List of shell commands to run to set up nodes. -setup_commands: [] - -# Custom commands that will be run on the head node after common setup. -head_setup_commands: [] - -# Custom commands that will be run on worker nodes after common setup. -worker_setup_commands: [] - -# Command to start ray on the head node. You don't need to change this. -head_start_ray_commands: [] - -# Command to start ray on worker nodes. You don't need to change this. -worker_start_ray_commands: [] + InstanceType: m5.xlarge diff --git a/release/long_running_tests/run.sh b/release/long_running_tests/run.sh index a376a8124..e48d7c899 100644 --- a/release/long_running_tests/run.sh +++ b/release/long_running_tests/run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -ray_version="" +ray_version="" commit="" ray_branch="" workload="" @@ -48,19 +48,20 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: $workload" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp37-cp37m-manylinux2014_x86_64.whl" -echo set-window-option -g mouse on > ~/.tmux.conf -echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc # Serve load testing tool -rm -r wrk || true && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && sudo cp wrk /usr/local/bin -pip install -U pip -unset RAY_ADDRESS -source activate tensorflow_p36 -conda remove -y --force wrapt || true +cur_dir=$(pwd) +cd /tmp && rm -rf wrk && git clone https://github.com/wg/wrk.git wrk && cd wrk && make -j && cp wrk /usr/local/bin +cd "$cur_dir" || exit + pip install --upgrade pip pip install -U tensorflow==1.14 -pip install -q -U "$wheel" Click +pip install -q -U "$wheel" pip install -q "ray[all]" "gym[atari]" -python "workloads/$workload.py" + +ray stop && sleep 2 + +unset RAY_ADDRESS +python "./workloads/$workload.py" diff --git a/release/long_running_tests/workloads/serve.py b/release/long_running_tests/workloads/serve.py index 6d404ac51..59b230776 100644 --- a/release/long_running_tests/workloads/serve.py +++ b/release/long_running_tests/workloads/serve.py @@ -11,7 +11,7 @@ from ray.cluster_utils import Cluster num_redis_shards = 1 redis_max_memory = 10**8 object_store_memory = 10**8 -num_nodes = 5 +num_nodes = 1 cluster = Cluster() for i in range(num_nodes): cluster.add_node( @@ -22,21 +22,20 @@ for i in range(num_nodes): resources={str(i): 2}, object_store_memory=object_store_memory, redis_max_memory=redis_max_memory, - dashboard_host="0.0.0.0") + dashboard_host="0.0.0.0", + ) ray.init(address=cluster.address, dashboard_host="0.0.0.0") client = serve.start() @serve.accept_batch -def echo(_): +def echo(requests_batch): time.sleep(0.01) # Sleep for 10ms - ray.show_in_dashboard( - str(serve.context.batch_size), key="Current batch size") - return ["hi {}".format(i) for i in range(serve.context.batch_size)] + return ["hi" for _ in range(len(requests_batch))] -config = {"num_replicas": 30, "max_batch_size": 16} +config = {"num_replicas": 7, "max_batch_size": 16} client.create_backend("echo:v1", echo, config=config) client.create_endpoint("echo", backend="echo:v1", route="/echo") @@ -53,12 +52,18 @@ time_to_run = "60m" while True: proc = subprocess.Popen( [ - "wrk", "-c", - str(connections), "-t", - str(num_threads), "-s", time_to_run, "http://127.0.0.1:8000/echo" + "wrk", + "-c", + str(connections), + "-t", + str(num_threads), + "-d", + time_to_run, + "http://127.0.0.1:8000/echo", ], stdout=PIPE, - stderr=PIPE) + stderr=PIPE, + ) print("started load testing") proc.wait() out, err = proc.communicate() diff --git a/release/long_running_tests/workloads/serve_failure.py b/release/long_running_tests/workloads/serve_failure.py index 129853289..534dcbda7 100644 --- a/release/long_running_tests/workloads/serve_failure.py +++ b/release/long_running_tests/workloads/serve_failure.py @@ -11,19 +11,20 @@ from ray.cluster_utils import Cluster num_redis_shards = 1 redis_max_memory = 10**8 object_store_memory = 10**8 -num_nodes = 5 -cpus_per_node = 2 +num_nodes = 1 +cpus_per_node = 10 cluster = Cluster() for i in range(num_nodes): cluster.add_node( redis_port=6379 if i == 0 else None, num_redis_shards=num_redis_shards if i == 0 else None, - num_cpus=2, + num_cpus=16, num_gpus=0, resources={str(i): 2}, object_store_memory=object_store_memory, redis_max_memory=redis_max_memory, - dashboard_host="0.0.0.0") + dashboard_host="0.0.0.0", + ) ray.init( address=cluster.address, dashboard_host="0.0.0.0", log_to_driver=False) diff --git a/release/rllib_tests/regression_tests/cluster.yaml b/release/rllib_tests/regression_tests/cluster.yaml index d0aa94e8c..f316e7dc2 100644 --- a/release/rllib_tests/regression_tests/cluster.yaml +++ b/release/rllib_tests/regression_tests/cluster.yaml @@ -3,6 +3,11 @@ cluster_name: ray-rllib-regression-tests min_workers: 0 max_workers: 0 +docker: + image: anyscale/ray-ml:latest-gpu + container_name: ray_container + pull_before_run: True + # Cloud-provider specific configuration. provider: type: aws @@ -16,24 +21,18 @@ auth: head_node: InstanceType: p3.16xlarge - ImageId: ami-07728e9e2742b0662 # Deep Learning AMI (Ubuntu 16.04) - - # Set primary volume to 25 GiB - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 100 - # List of shell commands to run to set up nodes. -setup_commands: [] +setup_commands: + - apt-get install -y libglib2.0-0 libcudnn7=7.6.5.32-1+cuda10.1 + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/release/rllib_tests/regression_tests/run.sh b/release/rllib_tests/regression_tests/run.sh index 4a692c2e5..f965f35f7 100755 --- a/release/rllib_tests/regression_tests/run.sh +++ b/release/rllib_tests/regression_tests/run.sh @@ -41,15 +41,18 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: ignored" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp37-cp37m-manylinux2014_x86_64.whl" conda uninstall -y terminado -source activate tensorflow_p36 && pip install -U pip -source activate tensorflow_p36 && pip install -U "$wheel" -source activate tensorflow_p36 && pip install "ray[rllib]" "ray[debug]" -source activate tensorflow_p36 && pip install torch==1.6 torchvision -source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0 +pip install -U pip +pip install -U "$wheel" +pip install "ray[rllib]" "ray" +pip install terminado +pip install torch==1.6 torchvision +pip install boto3==1.4.8 cython==0.29.0 + # Run tf learning tests. -source activate tensorflow_p36 && rllib train -f compact-regression-tests-tf.yaml +rllib train -f compact-regression-tests-tf.yaml + # Run torch learning tests. -source activate tensorflow_p36 && rllib train -f compact-regression-tests-torch.yaml +rllib train -f compact-regression-tests-torch.yaml diff --git a/release/rllib_tests/stress_tests/cluster.yaml b/release/rllib_tests/stress_tests/cluster.yaml index e4fd2d26d..a3d4d08da 100644 --- a/release/rllib_tests/stress_tests/cluster.yaml +++ b/release/rllib_tests/stress_tests/cluster.yaml @@ -1,105 +1,46 @@ -#################################################################### -# All nodes in this cluster will auto-terminate in 1 hour -#################################################################### - -# An unique identifier for the head node and workers of this cluster. cluster_name: ray-rllib-stress-tests -# The minimum number of workers nodes to launch in addition to the head -# node. This number should be >= 0. min_workers: 9 - -# The maximum number of workers nodes to launch in addition to the head -# node. This takes precedence over min_workers. max_workers: 9 -# The autoscaler will scale up the cluster to this target fraction of resource -# usage. For example, if a cluster of 10 nodes is 100% busy and -# target_utilization is 0.8, it would resize the cluster to 13. This fraction -# can be decreased to increase the aggressiveness of upscaling. -# This value must be less than 1.0 for scaling to happen. target_utilization_fraction: 0.8 +idle_timeout_minutes: 15 -# If a node is idle for this many minutes, it will be removed. -idle_timeout_minutes: 5 +docker: + image: anyscale/ray-ml:latest-gpu + container_name: ray_container + pull_before_run: True -# Cloud-provider specific configuration. provider: type: aws region: us-west-2 availability_zone: us-west-2a cache_stopped_nodes: False -# How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu -# By default Ray creates a new private keypair, but you can also use your own. -# If you do so, make sure to also set "KeyName" in the head and worker node -# configurations below. -# ssh_private_key: /path/to/your/key.pem -# Provider-specific config for the head node, e.g. instance type. By default -# Ray will auto-configure unspecified fields such as SubnetId and KeyName. -# For more documentation on available fields, see: -# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: InstanceType: p3.16xlarge - ImageId: ami-07728e9e2742b0662 # Deep Learning AMI (Ubuntu 16.04) - # Set primary volume to 25 GiB - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 100 - - # Additional options in the boto docs. - -# Provider-specific config for worker nodes, e.g. instance type. By default -# Ray will auto-configure unspecified fields such as SubnetId and KeyName. -# For more documentation on available fields, see: -# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances worker_nodes: - InstanceType: m4.16xlarge - ImageId: ami-07728e9e2742b0662 # Deep Learning AMI (Ubuntu 16.04) + InstanceType: m5.16xlarge - - # Set primary volume to 25 GiB - BlockDeviceMappings: - - DeviceName: /dev/sda1 - Ebs: - VolumeSize: 100 - - # Run workers on spot by default. Comment this out to use on-demand. - # InstanceMarketOptions: - # MarketType: spot - # Additional options can be found in the boto docs, e.g. - # SpotOptions: - # MaxPrice: MAX_HOURLY_PRICE - - # Additional options in the boto docs. - -# Files or directories to copy to the head and worker nodes. The format is a -# dictionary from REMOTE_PATH: LOCAL_PATH, e.g. file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", } -# List of shell commands to run to set up nodes. -setup_commands: [] +setup_commands: + - apt-get install -y libglib2.0-0 libcudnn7=7.6.5.32-1+cuda10.1 + - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl -# Custom commands that will be run on the head node after common setup. -head_setup_commands: [] - -# Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] -# Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml -# Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/release/rllib_tests/stress_tests/run.sh b/release/rllib_tests/stress_tests/run.sh index b038de9fb..c5c1eb676 100755 --- a/release/rllib_tests/stress_tests/run.sh +++ b/release/rllib_tests/stress_tests/run.sh @@ -42,14 +42,14 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: ignored" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp37-cp37m-manylinux2014_x86_64.whl" conda uninstall -y terminado -source activate tensorflow_p36 && pip install -U pip -source activate tensorflow_p36 && pip install -U "$wheel" -source activate tensorflow_p36 && pip install "ray[rllib]" "ray[debug]" -source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0 -source activate tensorflow_p36 +pip install -U pip +pip install -U "$wheel" +pip install "ray[rllib]" "ray" +pip install terminado +pip install boto3==1.4.8 cython==0.29.0 python3 wait_cluster.py diff --git a/release/rllib_tests/unit_gpu_tests/cluster.yaml b/release/rllib_tests/unit_gpu_tests/cluster.yaml index 2030bb2ac..23e59b788 100644 --- a/release/rllib_tests/unit_gpu_tests/cluster.yaml +++ b/release/rllib_tests/unit_gpu_tests/cluster.yaml @@ -3,6 +3,11 @@ cluster_name: ray-rllib-regression-tests min_workers: 0 max_workers: 0 +docker: + image: anyscale/ray-ml:latest-gpu + container_name: ray_container + pull_before_run: True + # Cloud-provider specific configuration. provider: type: aws @@ -16,7 +21,6 @@ auth: head_node: InstanceType: p2.xlarge # Cheaper 1GPU K80 instance - ImageId: ami-07728e9e2742b0662 # Deep Learning AMI (Ubuntu 16.04) # Set primary volume to 25 GiB BlockDeviceMappings: @@ -26,14 +30,15 @@ head_node: # List of shell commands to run to set up nodes. -setup_commands: [] +setup_commands: + - apt-get install -y libglib2.0-0 libcudnn7=7.6.5.32-1+cuda10.1 curl unzip gcc python3-dev # Command to start ray on the head node. You don't need to change this. head_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml # Command to start ray on worker nodes. You don't need to change this. worker_start_ray_commands: - - source activate tensorflow_p36 && ray stop - - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 + - ray stop + - ulimit -n 65536; OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/release/rllib_tests/unit_gpu_tests/requirements.txt b/release/rllib_tests/unit_gpu_tests/requirements.txt index a86eb47e6..4f8897539 100644 --- a/release/rllib_tests/unit_gpu_tests/requirements.txt +++ b/release/rllib_tests/unit_gpu_tests/requirements.txt @@ -1,6 +1,7 @@ ray[rllib] -ray[debug] +ray torch==1.6+cu101 torchvision==0.7.0+cu101 boto3==1.4.8 cython==0.29.0 +pytest diff --git a/release/rllib_tests/unit_gpu_tests/run.sh b/release/rllib_tests/unit_gpu_tests/run.sh index db468e789..ff93e5164 100755 --- a/release/rllib_tests/unit_gpu_tests/run.sh +++ b/release/rllib_tests/unit_gpu_tests/run.sh @@ -42,12 +42,33 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: ignored" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp37-cp37m-manylinux2014_x86_64.whl" conda uninstall -y terminado -source activate tensorflow_p36 && pip install -U pip -source activate tensorflow_p36 && pip install -U "$wheel" +pip install -U pip +pip install -U "$wheel" +pip install -U pytest +pip install terminado +pip install torch>=1.6 torchvision +pip install -U tensorflow-gpu + +if [ -z "$commit" ]; then + cob="origin/$ray_branch" +else + cob="$commit" +fi + +git clone https://github.com/ray-project/ray.git ray +pushd ray || true +git checkout "$cob" + +bash ./ci/travis/install-bazel.sh +BAZEL_PATH=$HOME/bin/bazel # Run all test cases, but with a forced num_gpus=1. # TODO: (sven) chose correct dir and run over all RLlib tests and example scripts! -source activate tensorflow_p36 && export RAY_FORCE_NUM_GPUS=1 && cd ~ && python -m pytest test_attention_net_learning.py +export RLLIB_NUM_GPUS=1 && $BAZEL_PATH test --config="ci $(./scripts/bazel_export_options)" --build_tests_only --test_tag_filters=examples_A,examples_B --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... +export RLLIB_NUM_GPUS=1 && $BAZEL_PATH test --config="ci $(./scripts/bazel_export_options)" --build_tests_only --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... +export RLLIB_NUM_GPUS=1 && $BAZEL_PATH test --config="ci $(./scripts/bazel_export_options)" --build_tests_only --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... +export RLLIB_NUM_GPUS=1 && $BAZEL_PATH test --config="ci $(./scripts/bazel_export_options)" --build_tests_only --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/... +popd || true diff --git a/release/stress_tests/cluster.yaml b/release/stress_tests/cluster.yaml index 6f7449ae0..61fe0e5f0 100644 --- a/release/stress_tests/cluster.yaml +++ b/release/stress_tests/cluster.yaml @@ -7,11 +7,11 @@ cluster_name: ray-stress-tests # The minimum number of workers nodes to launch in addition to the head # node. This number should be >= 0. -min_workers: 105 +min_workers: 100 # The maximum number of workers nodes to launch in addition to the head # node. This takes precedence over min_workers. -max_workers: 105 +max_workers: 100 # The autoscaler will scale up the cluster to this target fraction of resource # usage. For example, if a cluster of 10 nodes is 100% busy and @@ -44,7 +44,7 @@ auth: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances head_node: InstanceType: m4.16xlarge - ImageId: ami-06d51e91cea0dac8d # Ubuntu 18.04 + ImageId: ami-042d0d6196f494652 # Custom ami # Set primary volume to 25 GiB BlockDeviceMappings: @@ -60,7 +60,7 @@ head_node: # http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances worker_nodes: InstanceType: m4.large - ImageId: ami-06d51e91cea0dac8d # Ubuntu 18.04 + ImageId: ami-042d0d6196f494652 # Custom ami # Set primary volume to 25 GiB BlockDeviceMappings: @@ -77,32 +77,19 @@ worker_nodes: # Additional options in the boto docs. -# Files or directories to copy to the head and worker nodes. The format is a -# dictionary from REMOTE_PATH: LOCAL_PATH, e.g. -file_mounts: { -# "/path1/on/remote/machine": "/path1/on/local/machine", -# "/path2/on/remote/machine": "/path2/on/local/machine", -} - # List of shell commands to run to set up nodes. -setup_commands: [] +setup_commands: # Uncomment these if you want to build ray from source. # - sudo apt-get -qq update # - sudo apt-get install -y build-essential curl unzip - # Install Anaconda. - - wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true - - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true - - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc # # Build Ray. # - git clone https://github.com/ray-project/ray || true # - ray/ci/travis/install-bazel.sh - - pip install -U pip - - conda uninstall -y terminado || true + - pip install -U pip - pip install terminado - pip install boto3==1.4.8 cython==0.29.0 # - cd ray/python; git checkout master; git pull; pip install -e . --verbose - - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux2014_x86_64.whl" - + - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl" # Custom commands that will be run on the head node after common setup. head_setup_commands: [] diff --git a/release/stress_tests/requirements.txt b/release/stress_tests/requirements.txt index 0f026d879..e02756318 100644 --- a/release/stress_tests/requirements.txt +++ b/release/stress_tests/requirements.txt @@ -1 +1 @@ -ray[debug] \ No newline at end of file +ray diff --git a/release/stress_tests/run.sh b/release/stress_tests/run.sh index f58eacfc8..2541e9af5 100644 --- a/release/stress_tests/run.sh +++ b/release/stress_tests/run.sh @@ -38,20 +38,4 @@ echo "commit: $commit" echo "branch: $ray_branch" echo "workload: $workload" -wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl" - -# Install Anaconda. -wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true -bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p "$HOME/anaconda3" || true -# shellcheck disable=SC2016 -echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc - -conda uninstall -y terminado -source activate tensorflow_p36 && pip install -U pip -source activate tensorflow_p36 && pip install -U "$wheel" - -pip install -U pip -conda uninstall -y terminado || true -pip install terminado -pip install boto3==1.4.8 cython==0.29.0 python "workloads/$workload.py" diff --git a/rllib/BUILD b/rllib/BUILD index 8af609982..bd612d0ff 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -172,6 +172,7 @@ py_test( data = [ "tuned_examples/dqn/cartpole-simpleq.yaml", "tuned_examples/dqn/cartpole-dqn.yaml", + "tuned_examples/dqn/cartpole-dqn-softq.yaml", "tuned_examples/dqn/cartpole-dqn-param-noise.yaml", ], args = ["--yaml-dir=tuned_examples/dqn"] @@ -186,6 +187,7 @@ py_test( data = [ "tuned_examples/dqn/cartpole-simpleq.yaml", "tuned_examples/dqn/cartpole-dqn.yaml", + "tuned_examples/dqn/cartpole-dqn-softq.yaml", "tuned_examples/dqn/cartpole-dqn-param-noise.yaml", ], args = ["--yaml-dir=tuned_examples/dqn", "--torch"] @@ -1108,6 +1110,13 @@ py_test( # srcs = ["evaluation/tests/test_trajectory_view_api.py"] #) +py_test( + name = "evaluation/tests/test_rollout_worker", + tags = ["evaluation"], + size = "medium", + srcs = ["evaluation/tests/test_rollout_worker.py"] +) + # -------------------------------------------------------------------- # Optimizers and Memories @@ -1411,13 +1420,6 @@ py_test( args = ["TestRolloutLearntPolicy"] ) -py_test( - name = "tests/test_rollout_worker", - tags = ["tests_dir", "tests_dir_R"], - size = "medium", - srcs = ["tests/test_rollout_worker.py"] -) - py_test( name = "tests/test_supported_multi_agent_pg", main = "tests/test_supported_multi_agent.py", diff --git a/rllib/agents/a3c/a2c.py b/rllib/agents/a3c/a2c.py index 0a71a359c..e6ea0c356 100644 --- a/rllib/agents/a3c/a2c.py +++ b/rllib/agents/a3c/a2c.py @@ -38,17 +38,20 @@ def execution_plan(workers, config): # allowing for extremely large experience batches to be used. train_op = ( rollouts.combine( - ConcatBatches(min_batch_size=config["microbatch_size"])) + ConcatBatches( + min_batch_size=config["microbatch_size"], + count_steps_by=config["multiagent"]["count_steps_by"])) .for_each(ComputeGradients(workers)) # (grads, info) .batch(num_microbatches) # List[(grads, info)] .for_each(AverageGradients()) # (avg_grads, info) .for_each(ApplyGradients(workers))) else: # In normal mode, we execute one SGD step per each train batch. - train_op = rollouts \ - .combine(ConcatBatches( - min_batch_size=config["train_batch_size"])) \ - .for_each(TrainOneStep(workers)) + train_op = rollouts.combine( + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"][ + "count_steps_by"])).for_each(TrainOneStep(workers)) return StandardMetricsReporting(train_op, workers, config) diff --git a/rllib/agents/impala/impala.py b/rllib/agents/impala/impala.py index 7a09b1f9a..0bcddf4f3 100644 --- a/rllib/agents/impala/impala.py +++ b/rllib/agents/impala/impala.py @@ -221,7 +221,10 @@ def gather_experiences_directly(workers, config): replay_proportion=config["replay_proportion"])) \ .flatten() \ .combine( - ConcatBatches(min_batch_size=config["train_batch_size"])) + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )) return train_batches diff --git a/rllib/agents/marwil/marwil.py b/rllib/agents/marwil/marwil.py index 6aeb373c5..c4f88fdb8 100644 --- a/rllib/agents/marwil/marwil.py +++ b/rllib/agents/marwil/marwil.py @@ -56,7 +56,10 @@ def execution_plan(workers, config): replay_op = Replay(local_buffer=replay_buffer) \ .combine( - ConcatBatches(min_batch_size=config["train_batch_size"])) \ + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )) \ .for_each(TrainOneStep(workers)) train_op = Concurrently( diff --git a/rllib/agents/ppo/appo_tf_policy.py b/rllib/agents/ppo/appo_tf_policy.py index 693ec3da0..c45cfb0ba 100644 --- a/rllib/agents/ppo/appo_tf_policy.py +++ b/rllib/agents/ppo/appo_tf_policy.py @@ -13,12 +13,12 @@ from typing import Dict, List, Optional, Type, Union from ray.rllib.agents.impala import vtrace_tf as vtrace from ray.rllib.agents.impala.vtrace_tf_policy import _make_time_major, \ clip_gradients, choose_optimizer +from ray.rllib.agents.ppo.ppo_tf_policy import postprocess_ppo_gae from ray.rllib.evaluation.episode import MultiAgentEpisode from ray.rllib.evaluation.postprocessing import Postprocessing from ray.rllib.models.tf.tf_action_dist import Categorical from ray.rllib.policy.policy import Policy from ray.rllib.policy.sample_batch import SampleBatch -from ray.rllib.evaluation.postprocessing import compute_advantages from ray.rllib.policy.tf_policy_template import build_tf_policy from ray.rllib.policy.tf_policy import LearningRateSchedule, TFPolicy from ray.rllib.agents.ppo.ppo_tf_policy import KLCoeffMixin, ValueNetworkMixin @@ -338,31 +338,14 @@ def postprocess_trajectory( SampleBatch: The postprocessed, modified SampleBatch (or a new one). """ if not policy.config["vtrace"]: - completed = sample_batch["dones"][-1] - if completed: - last_r = 0.0 - else: - next_state = [] - for i in range(policy.num_state_tensors()): - next_state.append([sample_batch["state_out_{}".format(i)][-1]]) - last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1], - sample_batch[SampleBatch.ACTIONS][-1], - sample_batch[SampleBatch.REWARDS][-1], - *next_state) - batch = compute_advantages( - sample_batch, - last_r, - policy.config["gamma"], - policy.config["lambda"], - use_gae=policy.config["use_gae"], - use_critic=policy.config["use_critic"]) - else: - batch = sample_batch + sample_batch = postprocess_ppo_gae(policy, sample_batch, + other_agent_batches, episode) + # TODO: (sven) remove this del once we have trajectory view API fully in # place. - del batch.data["new_obs"] # not used, so save some bandwidth + del sample_batch.data["new_obs"] # not used, so save some bandwidth - return batch + return sample_batch def add_values(policy): diff --git a/rllib/agents/ppo/ppo.py b/rllib/agents/ppo/ppo.py index c8f6db43f..026988201 100644 --- a/rllib/agents/ppo/ppo.py +++ b/rllib/agents/ppo/ppo.py @@ -38,7 +38,7 @@ DEFAULT_CONFIG = with_common_config({ # If true, use the Generalized Advantage Estimator (GAE) # with a value function, see https://arxiv.org/pdf/1506.02438.pdf. "use_gae": True, - # The GAE(lambda) parameter. + # The GAE (lambda) parameter. "lambda": 1.0, # Initial coefficient for KL divergence. "kl_coeff": 0.2, @@ -244,7 +244,10 @@ def execution_plan(workers: WorkerSet, SelectExperiences(workers.trainable_policies())) # Concatenate the SampleBatches into one. rollouts = rollouts.combine( - ConcatBatches(min_batch_size=config["train_batch_size"])) + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )) # Standardize advantages. rollouts = rollouts.for_each(StandardizeFields(["advantages"])) diff --git a/rllib/agents/ppo/ppo_tf_policy.py b/rllib/agents/ppo/ppo_tf_policy.py index 5a8ef7b80..29266dfcc 100644 --- a/rllib/agents/ppo/ppo_tf_policy.py +++ b/rllib/agents/ppo/ppo_tf_policy.py @@ -193,13 +193,22 @@ def postprocess_ppo_gae( last_r = 0.0 # Trajectory has been truncated -> last r=VF estimate of last obs. else: - next_state = [] - for i in range(policy.num_state_tensors()): - next_state.append(sample_batch["state_out_{}".format(i)][-1]) - last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1], - sample_batch[SampleBatch.ACTIONS][-1], - sample_batch[SampleBatch.REWARDS][-1], - *next_state) + # Input dict is provided to us automatically via the Model's + # requirements. It's a single-timestep (last one in trajectory) + # input_dict. + if policy.config["_use_trajectory_view_api"]: + # Create an input dict according to the Model's requirements. + input_dict = policy.model.get_input_dict(sample_batch, index=-1) + last_r = policy._value(**input_dict) + # TODO: (sven) Remove once trajectory view API is all-algo default. + else: + next_state = [] + for i in range(policy.num_state_tensors()): + next_state.append(sample_batch["state_out_{}".format(i)][-1]) + last_r = policy._value(sample_batch[SampleBatch.NEXT_OBS][-1], + sample_batch[SampleBatch.ACTIONS][-1], + sample_batch[SampleBatch.REWARDS][-1], + *next_state) # Adds the policy logits, VF preds, and advantages to the batch, # using GAE ("generalized advantage estimation") or not. @@ -208,7 +217,9 @@ def postprocess_ppo_gae( last_r, policy.config["gamma"], policy.config["lambda"], - use_gae=policy.config["use_gae"]) + use_gae=policy.config["use_gae"], + use_critic=policy.config.get("use_critic", True)) + return batch @@ -292,25 +303,40 @@ class ValueNetworkMixin: # observation. if config["use_gae"]: - @make_tf_callable(self.get_session()) - def value(ob, prev_action, prev_reward, *state): - model_out, _ = self.model({ - SampleBatch.CUR_OBS: tf.convert_to_tensor([ob]), - SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( - [prev_action]), - SampleBatch.PREV_REWARDS: tf.convert_to_tensor( - [prev_reward]), - "is_training": tf.convert_to_tensor([False]), - }, [tf.convert_to_tensor([s]) for s in state], - tf.convert_to_tensor([1])) - # [0] = remove the batch dim. - return self.model.value_function()[0] + # Input dict is provided to us automatically via the Model's + # requirements. It's a single-timestep (last one in trajectory) + # input_dict. + if config["_use_trajectory_view_api"]: + + @make_tf_callable(self.get_session()) + def value(**input_dict): + model_out, _ = self.model.from_batch( + input_dict, is_training=False) + # [0] = remove the batch dim. + return self.model.value_function()[0] + + # TODO: (sven) Remove once trajectory view API is all-algo default. + else: + + @make_tf_callable(self.get_session()) + def value(ob, prev_action, prev_reward, *state): + model_out, _ = self.model({ + SampleBatch.CUR_OBS: tf.convert_to_tensor([ob]), + SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( + [prev_action]), + SampleBatch.PREV_REWARDS: tf.convert_to_tensor( + [prev_reward]), + "is_training": tf.convert_to_tensor([False]), + }, [tf.convert_to_tensor([s]) for s in state], + tf.convert_to_tensor([1])) + # [0] = remove the batch dim. + return self.model.value_function()[0] # When not doing GAE, we do not require the value function's output. else: @make_tf_callable(self.get_session()) - def value(ob, prev_action, prev_reward, *state): + def value(*args, **kwargs): return tf.constant(0.0) self._value = value diff --git a/rllib/agents/ppo/ppo_torch_policy.py b/rllib/agents/ppo/ppo_torch_policy.py index 58637fa0a..a268e7487 100644 --- a/rllib/agents/ppo/ppo_torch_policy.py +++ b/rllib/agents/ppo/ppo_torch_policy.py @@ -210,22 +210,36 @@ class ValueNetworkMixin: # When doing GAE, we need the value function estimate on the # observation. if config["use_gae"]: + # Input dict is provided to us automatically via the Model's + # requirements. It's a single-timestep (last one in trajectory) + # input_dict. + if config["_use_trajectory_view_api"]: - def value(ob, prev_action, prev_reward, *state): - model_out, _ = self.model({ - SampleBatch.CUR_OBS: convert_to_torch_tensor( - np.asarray([ob]), self.device), - SampleBatch.PREV_ACTIONS: convert_to_torch_tensor( - np.asarray([prev_action]), self.device), - SampleBatch.PREV_REWARDS: convert_to_torch_tensor( - np.asarray([prev_reward]), self.device), - "is_training": False, - }, [ - convert_to_torch_tensor(np.asarray([s]), self.device) - for s in state - ], convert_to_torch_tensor(np.asarray([1]), self.device)) - # [0] = remove the batch dim. - return self.model.value_function()[0] + def value(**input_dict): + model_out, _ = self.model.from_batch( + convert_to_torch_tensor(input_dict, self.device), + is_training=False) + # [0] = remove the batch dim. + return self.model.value_function()[0] + + # TODO: (sven) Remove once trajectory view API is all-algo default. + else: + + def value(ob, prev_action, prev_reward, *state): + model_out, _ = self.model({ + SampleBatch.CUR_OBS: convert_to_torch_tensor( + np.asarray([ob]), self.device), + SampleBatch.PREV_ACTIONS: convert_to_torch_tensor( + np.asarray([prev_action]), self.device), + SampleBatch.PREV_REWARDS: convert_to_torch_tensor( + np.asarray([prev_reward]), self.device), + "is_training": False, + }, [ + convert_to_torch_tensor(np.asarray([s]), self.device) + for s in state + ], convert_to_torch_tensor(np.asarray([1]), self.device)) + # [0] = remove the batch dim. + return self.model.value_function()[0] # When not doing GAE, we do not require the value function's output. else: diff --git a/rllib/agents/ppo/tests/test_ppo.py b/rllib/agents/ppo/tests/test_ppo.py index c00cd36ba..b4259c144 100644 --- a/rllib/agents/ppo/tests/test_ppo.py +++ b/rllib/agents/ppo/tests/test_ppo.py @@ -73,7 +73,7 @@ class TestPPO(unittest.TestCase): def test_ppo_compilation_and_lr_schedule(self): """Test whether a PPOTrainer can be built with all frameworks.""" config = copy.deepcopy(ppo.DEFAULT_CONFIG) - # for checking lr-schedule correctness + # For checking lr-schedule correctness. config["callbacks"] = MyCallbacks config["num_workers"] = 1 diff --git a/rllib/agents/qmix/model.py b/rllib/agents/qmix/model.py index 42e55fe7b..313e91ecb 100644 --- a/rllib/agents/qmix/model.py +++ b/rllib/agents/qmix/model.py @@ -1,9 +1,6 @@ -from gym.spaces import Box - from ray.rllib.models.modelv2 import ModelV2 from ray.rllib.models.preprocessors import get_preprocessor from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 -from ray.rllib.policy.view_requirement import ViewRequirement from ray.rllib.utils.annotations import override from ray.rllib.utils.framework import try_import_torch @@ -25,17 +22,13 @@ class RNNModel(TorchModelV2, nn.Module): self.fc2 = nn.Linear(self.rnn_hidden_dim, num_outputs) self.n_agents = model_config["n_agents"] - self.inference_view_requirements.update({ - "state_in_0": ViewRequirement( - "state_out_0", - data_rel_pos=-1, - space=Box(-1.0, 1.0, (self.n_agents, self.rnn_hidden_dim))) - }) - @override(ModelV2) def get_initial_state(self): # Place hidden states on same device as model. - return [self.fc1.weight.new(1, self.rnn_hidden_dim).zero_().squeeze(0)] + return [ + self.fc1.weight.new(self.n_agents, + self.rnn_hidden_dim).zero_().squeeze(0) + ] @override(ModelV2) def forward(self, input_dict, hidden_state, seq_lens): diff --git a/rllib/agents/qmix/qmix.py b/rllib/agents/qmix/qmix.py index 7d64680f5..c2584378d 100644 --- a/rllib/agents/qmix/qmix.py +++ b/rllib/agents/qmix/qmix.py @@ -109,7 +109,10 @@ def execution_plan(workers, config): train_op = Replay(local_buffer=replay_buffer) \ .combine( - ConcatBatches(min_batch_size=config["train_batch_size"])) \ + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"] + )) \ .for_each(TrainOneStep(workers)) \ .for_each(UpdateTargetNetwork( workers, config["target_network_update_freq"])) diff --git a/rllib/agents/qmix/qmix_policy.py b/rllib/agents/qmix/qmix_policy.py index a22518a04..6b533378a 100644 --- a/rllib/agents/qmix/qmix_policy.py +++ b/rllib/agents/qmix/qmix_policy.py @@ -215,9 +215,6 @@ class QMixTorchPolicy(Policy): name="target_model", default_model=RNNModel).to(self.device) - # Combine view_requirements for Model and Policy. - self.view_requirements.update(self.model.inference_view_requirements) - self.exploration = self._create_exploration() # Setup the mixer network. diff --git a/rllib/agents/trainer.py b/rllib/agents/trainer.py index b5751e264..c57ff0b67 100644 --- a/rllib/agents/trainer.py +++ b/rllib/agents/trainer.py @@ -75,10 +75,18 @@ COMMON_CONFIG: TrainerConfigDict = { # The dataflow here can vary per algorithm. For example, PPO further # divides the train batch into minibatches for multi-epoch SGD. "rollout_fragment_length": 200, - # Whether to rollout "complete_episodes" or "truncate_episodes" to - # `rollout_fragment_length` length unrolls. Episode truncation guarantees - # evenly sized batches, but increases variance as the reward-to-go will - # need to be estimated at truncation boundaries. + # How to build per-Sampler (RolloutWorker) batches, which are then + # usually concat'd to form the train batch. Note that "steps" below can + # mean different things (either env- or agent-steps) and depends on the + # `count_steps_by` (multiagent) setting below. + # truncate_episodes: Each produced batch (when calling + # RolloutWorker.sample()) will contain exactly `rollout_fragment_length` + # steps. This mode guarantees evenly sized batches, but increases + # variance as the future return must now be estimated at truncation + # boundaries. + # complete_episodes: Each unroll happens exactly over one episode, from + # beginning to end. Data collection will not stop unless the episode + # terminates or a configured horizon (hard or soft) is hit. "batch_mode": "truncate_episodes", # === Settings for the Trainer process === @@ -357,6 +365,13 @@ COMMON_CONFIG: TrainerConfigDict = { # agents it controls at that timestep. When replay_mode=independent, # transitions are replayed independently per policy. "replay_mode": "independent", + # Which metric to use as the "batch size" when building a + # MultiAgentBatch. The two supported values are: + # env_steps: Count each time the env is "stepped" (no matter how many + # multi-agent actions are passed/how many multi-agent observations + # have been returned in the previous step). + # agent_steps: Count each individual agent step as one step. + "count_steps_by": "env_steps", }, # === Logger === @@ -1081,6 +1096,20 @@ class Trainer(Trainable): config["model"]["lstm_use_prev_action"] = prev_a_r config["model"]["lstm_use_prev_reward"] = prev_a_r + # Check batching/sample collection settings. + if config["batch_mode"] not in [ + "truncate_episodes", "complete_episodes" + ]: + raise ValueError("`batch_mode` must be one of [truncate_episodes|" + "complete_episodes]! Got {}".format( + config["batch_mode"])) + + if config["multiagent"].get("count_steps_by", "env_steps") not in \ + ["env_steps", "agent_steps"]: + raise ValueError( + "`count_steps_by` must be one of [env_steps|agent_steps]! " + "Got {}".format(config["multiagent"]["count_steps_by"])) + def _try_recover(self): """Try to identify and remove any unhealthy workers. diff --git a/rllib/agents/trainer_template.py b/rllib/agents/trainer_template.py index f3c5d4c1c..b896958b6 100644 --- a/rllib/agents/trainer_template.py +++ b/rllib/agents/trainer_template.py @@ -22,10 +22,11 @@ def default_execution_plan(workers: WorkerSet, config: TrainerConfigDict): # Combine experiences batches until we hit `train_batch_size` in size. # Then, train the policy on those experiences and update the workers. - train_op = rollouts \ - .combine(ConcatBatches( - min_batch_size=config["train_batch_size"])) \ - .for_each(TrainOneStep(workers)) + train_op = rollouts.combine( + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )).for_each(TrainOneStep(workers)) # Add on the standard episode reward, etc. metrics reporting. This returns # a LocalIterator[metrics_dict] representing metrics for each train step. @@ -64,7 +65,7 @@ def build_trainer( Optional callable that takes the config to check for correctness. It may mutate the config as needed. default_policy (Optional[Type[Policy]]): The default Policy class to - use. + use if `get_policy_class` returns None. get_policy_class (Optional[Callable[ TrainerConfigDict, Optional[Type[Policy]]]]): Optional callable that takes a config and returns the policy class or None. If None diff --git a/rllib/contrib/alpha_zero/core/alpha_zero_trainer.py b/rllib/contrib/alpha_zero/core/alpha_zero_trainer.py index 27315108b..4af65eba6 100644 --- a/rllib/contrib/alpha_zero/core/alpha_zero_trainer.py +++ b/rllib/contrib/alpha_zero/core/alpha_zero_trainer.py @@ -164,11 +164,12 @@ def execution_plan(workers, config): rollouts = ParallelRollouts(workers, mode="bulk_sync") if config["simple_optimizer"]: - train_op = rollouts \ - .combine(ConcatBatches( - min_batch_size=config["train_batch_size"])) \ - .for_each(TrainOneStep( - workers, num_sgd_iter=config["num_sgd_iter"])) + train_op = rollouts.combine( + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )).for_each( + TrainOneStep(workers, num_sgd_iter=config["num_sgd_iter"])) else: replay_buffer = SimpleReplayBuffer(config["buffer_size"]) @@ -178,7 +179,10 @@ def execution_plan(workers, config): replay_op = Replay(local_buffer=replay_buffer) \ .filter(WaitUntilTimestepsElapsed(config["learning_starts"])) \ .combine( - ConcatBatches(min_batch_size=config["train_batch_size"])) \ + ConcatBatches( + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )) \ .for_each(TrainOneStep( workers, num_sgd_iter=config["num_sgd_iter"])) diff --git a/rllib/contrib/maddpg/maddpg_policy.py b/rllib/contrib/maddpg/maddpg_policy.py index 35a4b0cce..913c3f530 100644 --- a/rllib/contrib/maddpg/maddpg_policy.py +++ b/rllib/contrib/maddpg/maddpg_policy.py @@ -28,7 +28,7 @@ class MADDPGPostprocessing: other_agent_batches=None, episode=None): # FIXME: Get done from info is required since agentwise done is not - # supported now. + # supported now. sample_batch.data[SampleBatch.DONES] = self.get_done_from_info( sample_batch.data[SampleBatch.INFOS]) @@ -251,6 +251,9 @@ class MADDPGTFPolicy(MADDPGPostprocessing, TFPolicy): loss_inputs=loss_inputs, dist_inputs=actor_feature) + del self.view_requirements["prev_actions"] + del self.view_requirements["prev_rewards"] + self.sess.run(tf1.global_variables_initializer()) # Hard initial update diff --git a/rllib/env/atari_wrappers.py b/rllib/env/atari_wrappers.py index 347bc6167..48ba221b3 100644 --- a/rllib/env/atari_wrappers.py +++ b/rllib/env/atari_wrappers.py @@ -278,7 +278,7 @@ def wrap_deepmind(env, dim=84, framestack=True): """ env = MonitorEnv(env) env = NoopResetEnv(env, noop_max=30) - if "NoFrameskip" in env.spec.id: + if env.spec is not None and "NoFrameskip" in env.spec.id: env = MaxAndSkipEnv(env, skip=4) env = EpisodicLifeEnv(env) if "FIRE" in env.unwrapped.get_action_meanings(): diff --git a/rllib/env/policy_client.py b/rllib/env/policy_client.py index 7b6ce4306..232f74f1a 100644 --- a/rllib/env/policy_client.py +++ b/rllib/env/policy_client.py @@ -269,7 +269,8 @@ class _LocalInferenceThread(threading.Thread): if isinstance(samples, MultiAgentBatch): logger.info( "Sending batch of {} env steps ({} agent steps) to " - "server.".format(samples.count, samples.total())) + "server.".format(samples.env_steps(), + samples.agent_steps())) else: logger.info( "Sending batch of {} steps back to server.".format( diff --git a/rllib/evaluation/collectors/sample_collector.py b/rllib/evaluation/collectors/sample_collector.py index 6ebc3d097..da188e938 100644 --- a/rllib/evaluation/collectors/sample_collector.py +++ b/rllib/evaluation/collectors/sample_collector.py @@ -110,11 +110,37 @@ class _SampleCollector(metaclass=ABCMeta): @abstractmethod def total_env_steps(self) -> int: - """Returns total number of steps taken in the env (sum of all agents). + """Returns total number of env-steps taken so far. + + Thereby, a step in an N-agent multi-agent environment counts as only 1 + for this metric. The returned count contains everything that has not + been built yet (and returned as MultiAgentBatches by the + `try_build_truncated_episode_multi_agent_batch` or + `postprocess_episode(build=True)` methods). After such build, this + counter is reset to 0. Returns: - int: The number of steps taken in total in the environment over all - agents. + int: The number of env-steps taken in total in the environment(s) + so far. + """ + raise NotImplementedError + + @abstractmethod + def total_agent_steps(self) -> int: + """Returns total number of (individual) agent-steps taken so far. + + Thereby, a step in an N-agent multi-agent environment counts as N. + If less than N agents have stepped (because some agents were not + required to send actions), the count will be increased by less than N. + The returned count contains everything that has not been built yet + (and returned as MultiAgentBatches by the + `try_build_truncated_episode_multi_agent_batch` or + `postprocess_episode(build=True)` methods). After such build, this + counter is reset to 0. + + Returns: + int: The number of (individual) agent-steps taken in total in the + environment(s) so far. """ raise NotImplementedError @@ -191,7 +217,7 @@ class _SampleCollector(metaclass=ABCMeta): postprocessor. This is usually called to collect samples for policy training. If not enough data has been collected yet (`rollout_fragment_length`), - returns None. + returns an empty list. Returns: List[Union[MultiAgentBatch, SampleBatch]]: Returns a (possibly diff --git a/rllib/evaluation/collectors/simple_list_collector.py b/rllib/evaluation/collectors/simple_list_collector.py index a6be8c72f..efcadf32f 100644 --- a/rllib/evaluation/collectors/simple_list_collector.py +++ b/rllib/evaluation/collectors/simple_list_collector.py @@ -1,4 +1,5 @@ import collections +from gym.spaces import Space import logging import numpy as np from typing import Any, List, Dict, Tuple, TYPE_CHECKING, Union @@ -8,12 +9,11 @@ from ray.rllib.evaluation.collectors.sample_collector import _SampleCollector from ray.rllib.evaluation.episode import MultiAgentEpisode from ray.rllib.policy.policy import Policy from ray.rllib.policy.sample_batch import SampleBatch, MultiAgentBatch -from ray.rllib.policy.view_requirement import ViewRequirement from ray.rllib.utils.annotations import override from ray.rllib.utils.debug import summarize -from ray.rllib.utils.typing import AgentID, EpisodeID, EnvID, PolicyID, \ - TensorType from ray.rllib.utils.framework import try_import_tf, try_import_torch +from ray.rllib.utils.typing import AgentID, EpisodeID, EnvID, PolicyID, \ + TensorType, ViewRequirementsDict from ray.util.debug import log_once _, tf, _ = try_import_tf() @@ -48,13 +48,13 @@ class _AgentCollector: def __init__(self, shift_before: int = 0): self.shift_before = max(shift_before, 1) self.buffers: Dict[str, List] = {} + self.episode_id = None # The simple timestep count for this agent. Gets increased by one # each time a (non-initial!) observation is added. - self.count = 0 + self.agent_steps = 0 def add_init_obs(self, episode_id: EpisodeID, agent_index: int, - env_id: EnvID, t: int, init_obs: TensorType, - view_requirements: Dict[str, ViewRequirement]) -> None: + env_id: EnvID, t: int, init_obs: TensorType) -> None: """Adds an initial observation (after reset) to the Agent's trajectory. Args: @@ -67,19 +67,17 @@ class _AgentCollector: ts=-1(!), then an action/reward/next-obs at t=0, etc.. init_obs (TensorType): The initial observation tensor (after `env.reset()`). - view_requirements (Dict[str, ViewRequirements]) """ if SampleBatch.OBS not in self.buffers: self._build_buffers( single_row={ SampleBatch.OBS: init_obs, - SampleBatch.EPS_ID: episode_id, SampleBatch.AGENT_INDEX: agent_index, "env_id": env_id, "t": t, }) self.buffers[SampleBatch.OBS].append(init_obs) - self.buffers[SampleBatch.EPS_ID].append(episode_id) + self.episode_id = episode_id self.buffers[SampleBatch.AGENT_INDEX].append(agent_index) self.buffers["env_id"].append(env_id) self.buffers["t"].append(t) @@ -97,15 +95,19 @@ class _AgentCollector: assert SampleBatch.OBS not in values values[SampleBatch.OBS] = values[SampleBatch.NEXT_OBS] del values[SampleBatch.NEXT_OBS] + # Make sure EPS_ID stays the same for this agent. Usually, it should + # not be part of `values` anyways. + if SampleBatch.EPS_ID in values: + assert values[SampleBatch.EPS_ID] == self.episode_id + del values[SampleBatch.EPS_ID] for k, v in values.items(): if k not in self.buffers: self._build_buffers(single_row=values) self.buffers[k].append(v) - self.count += 1 + self.agent_steps += 1 - def build(self, view_requirements: Dict[str, ViewRequirement]) -> \ - SampleBatch: + def build(self, view_requirements: ViewRequirementsDict) -> SampleBatch: """Builds a SampleBatch from the thus-far collected agent data. If the episode/trajectory has no DONE=True at the end, will copy @@ -115,32 +117,29 @@ class _AgentCollector: by a Policy. Args: - view_requirements (Dict[str, ViewRequirement]: The view + view_requirements (ViewRequirementsDict): The view requirements dict needed to build the SampleBatch from the raw buffers (which may have data shifts as well as mappings from view-col to data-col in them). + Returns: SampleBatch: The built SampleBatch for this agent, ready to go into postprocessing. """ - # TODO: measure performance gains when using a UsageTrackingDict - # instead of a SampleBatch for postprocessing (this would eliminate - # copies (for creating this SampleBatch) of many unused columns for - # no reason (not used by postprocessor)). - batch_data = {} np_data = {} for view_col, view_req in view_requirements.items(): # Create the batch of data from the different buffers. data_col = view_req.data_col or view_col + # Some columns don't exist yet (get created during postprocessing). # -> skip. if data_col not in self.buffers: continue # OBS are already shifted by -1 (the initial obs starts one ts # before all other data columns). - shift = view_req.data_rel_pos - \ + shift = view_req.shift - \ (1 if data_col == SampleBatch.OBS else 0) if data_col not in np_data: np_data[data_col] = to_float_np_array(self.buffers[data_col]) @@ -161,8 +160,12 @@ class _AgentCollector: data = np_data[data_col][self.shift_before + shift:shift] if len(data) > 0: batch_data[view_col] = data + batch = SampleBatch(batch_data) + # Add EPS_ID and UNROLL_ID to batch. + batch.data[SampleBatch.EPS_ID] = np.repeat(self.episode_id, + batch.count) if SampleBatch.UNROLL_ID not in batch.data: # TODO: (sven) Once we have the additional # model.preprocess_train_batch in place (attention net PR), we @@ -180,7 +183,7 @@ class _AgentCollector: if self.shift_before > 0: for k, data in self.buffers.items(): self.buffers[k] = data[-self.shift_before:] - self.count = 0 + self.agent_steps = 0 return batch @@ -200,7 +203,7 @@ class _AgentCollector: ] else 0) # Python primitive or dict (e.g. INFOs). if isinstance(data, (int, float, bool, str, dict)): - self.buffers[col] = [0 for _ in range(shift)] + self.buffers[col] = [data for _ in range(shift)] # np.ndarray, torch.Tensor, or tf.Tensor. else: shape = data.shape @@ -235,31 +238,30 @@ class _PolicyCollector: # NOTE: This is not an env-step count (across n agents). AgentA and # agentB, both using this policy, acting in the same episode and both # doing n steps would increase the count by 2*n. - self.count = 0 + self.agent_steps = 0 def add_postprocessed_batch_for_training( self, batch: SampleBatch, - view_requirements: Dict[str, ViewRequirement]) -> None: + view_requirements: ViewRequirementsDict) -> None: """Adds a postprocessed SampleBatch (single agent) to our buffers. Args: - batch (SampleBatch): A single agent (one trajectory) SampleBatch - to be added to the Policy's buffers. - view_requirements (Dict[str, ViewRequirement]: The view + batch (SampleBatch): An individual agent's (one trajectory) + SampleBatch to be added to the Policy's buffers. + view_requirements (ViewRequirementsDict): The view requirements for the policy. This is so we know, whether a view-column needs to be copied at all (not needed for training). """ for view_col, data in batch.items(): - # TODO(ekl) how do we handle this for policies that don't extend - # Torch / TF Policy template (no inference of view reqs)? - # Skip columns that are not used for training. - # if view_col not in view_requirements or \ - # not view_requirements[view_col].used_for_training: - # continue - self.buffers[view_col].extend(data) + # 1) If col is not in view_requirements, we must have a direct + # child of the base Policy that doesn't do auto-view req creation. + # 2) Col is in view-reqs and needed for training. + if view_col not in view_requirements or \ + view_requirements[view_col].used_for_training: + self.buffers[view_col].extend(data) # Add the agent's trajectory length to our count. - self.count += batch.count + self.agent_steps += batch.count def build(self): """Builds a SampleBatch for this policy from the collected data. @@ -275,8 +277,8 @@ class _PolicyCollector: assert SampleBatch.UNROLL_ID in batch.data # Clear buffers for future samples. self.buffers.clear() - # Reset count to 0. - self.count = 0 + # Reset agent steps to 0. + self.agent_steps = 0 return batch @@ -286,7 +288,11 @@ class _PolicyCollectorGroup: pid: _PolicyCollector() for pid in policy_map.keys() } - self.count = 0 + # Total env-steps (1 env-step=up to N agents stepped). + self.env_steps = 0 + # Total agent steps (1 agent-step=1 individual agent (out of N) + # stepped). + self.agent_steps = 0 class _SimpleListCollector(_SampleCollector): @@ -303,7 +309,8 @@ class _SimpleListCollector(_SampleCollector): clip_rewards: Union[bool, float], callbacks: "DefaultCallbacks", multiple_episodes_in_batch: bool = True, - rollout_fragment_length: int = 200): + rollout_fragment_length: int = 200, + count_steps_by: str = "env_steps"): """Initializes a _SimpleListCollector instance. Args: @@ -312,6 +319,10 @@ class _SimpleListCollector(_SampleCollector): clip_rewards (Union[bool, float]): Whether to clip rewards before postprocessing (at +/-1.0) or the actual value to +/- clip. callbacks (DefaultCallbacks): RLlib callbacks. + multiple_episodes_in_batch (bool): Whether it's allowed to pack + multiple episodes into the same built batch. + rollout_fragment_length (int): The + """ self.policy_map = policy_map @@ -319,6 +330,7 @@ class _SimpleListCollector(_SampleCollector): self.callbacks = callbacks self.multiple_episodes_in_batch = multiple_episodes_in_batch self.rollout_fragment_length = rollout_fragment_length + self.count_steps_by = count_steps_by self.large_batch_threshold: int = max( 1000, rollout_fragment_length * 10) if rollout_fragment_length != float("inf") else 5000 @@ -338,8 +350,10 @@ class _SimpleListCollector(_SampleCollector): self.forward_pass_size = {pid: 0 for pid in policy_map.keys()} # Maps episode ID to the (non-built) env steps taken in this episode. - self.episode_steps: Dict[EpisodeID, int] = \ - collections.defaultdict(int) + self.episode_steps: Dict[EpisodeID, int] = collections.defaultdict(int) + # Maps episode ID to the (non-built) individual agent steps in this + # episode. + self.agent_steps: Dict[EpisodeID, int] = collections.defaultdict(int) # Maps episode ID to MultiAgentEpisode. self.episodes: Dict[EpisodeID, MultiAgentEpisode] = {} @@ -349,15 +363,17 @@ class _SimpleListCollector(_SampleCollector): self.episode_steps[episode_id] += 1 episode.length += 1 assert episode.batch_builder is not None - env_steps = episode.batch_builder.count - num_observations = sum( - c.count for c in episode.batch_builder.policy_collectors.values()) + env_steps = episode.batch_builder.env_steps + num_individual_observations = sum( + c.agent_steps + for c in episode.batch_builder.policy_collectors.values()) - if num_observations > self.large_batch_threshold and \ + if num_individual_observations > self.large_batch_threshold and \ log_once("large_batch_warning"): logger.warning( "More than {} observations in {} env steps for " - "episode {} ".format(num_observations, env_steps, episode_id) + + "episode {} ".format(num_individual_observations, env_steps, + episode_id) + "are buffered in the sampler. If this is more than you " "expected, check that that you set a horizon on your " "environment correctly and that it terminates at some point. " @@ -380,9 +396,6 @@ class _SimpleListCollector(_SampleCollector): self.agent_key_to_policy_id[agent_key] = policy_id else: assert self.agent_key_to_policy_id[agent_key] == policy_id - policy = self.policy_map[policy_id] - view_reqs = policy.model.inference_view_requirements if \ - getattr(policy, "model", None) else policy.view_requirements # Add initial obs to Trajectory. assert agent_key not in self.agent_collectors @@ -393,8 +406,7 @@ class _SimpleListCollector(_SampleCollector): agent_index=episode._agent_index(agent_id), env_id=env_id, t=t, - init_obs=init_obs, - view_requirements=view_reqs) + init_obs=init_obs) self.episodes[episode.episode_id] = episode if episode.batch_builder is None: @@ -414,6 +426,8 @@ class _SimpleListCollector(_SampleCollector): assert self.agent_key_to_policy_id[agent_key] == policy_id assert agent_key in self.agent_collectors + self.agent_steps[episode_id] += 1 + # Include the current agent id for multi-agent algorithms. if agent_id != _DUMMY_AGENT_ID: values["agent_id"] = agent_id @@ -426,7 +440,18 @@ class _SimpleListCollector(_SampleCollector): @override(_SampleCollector) def total_env_steps(self) -> int: - return sum(a.count for a in self.agent_collectors.values()) + # Add the non-built ongoing-episode env steps + the already built + # env-steps. + return sum(self.episode_steps.values()) + sum( + pg.env_steps for pg in self.policy_collector_groups.values()) + + @override(_SampleCollector) + def total_agent_steps(self) -> int: + # Add the non-built ongoing-episode agent steps (still in the agent + # collectors) + the already built agent steps. + return sum(a.agent_steps for a in self.agent_collectors.values()) + \ + sum(pg.agent_steps for pg in + self.policy_collector_groups.values()) @override(_SampleCollector) def get_inference_input_dict(self, policy_id: PolicyID) -> \ @@ -442,17 +467,22 @@ class _SimpleListCollector(_SampleCollector): # Create the batch of data from the different buffers. data_col = view_req.data_col or view_col time_indices = \ - view_req.data_rel_pos - ( + view_req.shift - ( 1 if data_col in [SampleBatch.OBS, "t", "env_id", - SampleBatch.EPS_ID, SampleBatch.AGENT_INDEX] else 0) data_list = [] for k in keys: - if data_col not in buffers[k]: - self.agent_collectors[k]._build_buffers({ - data_col: view_req.space.sample() - }) - data_list.append(buffers[k][data_col][time_indices]) + if data_col == SampleBatch.EPS_ID: + data_list.append(self.agent_collectors[k].episode_id) + else: + if data_col not in buffers[k]: + fill_value = np.zeros_like(view_req.space.sample()) \ + if isinstance(view_req.space, Space) else \ + view_req.space + self.agent_collectors[k]._build_buffers({ + data_col: fill_value + }) + data_list.append(buffers[k][data_col][time_indices]) input_dict[view_col] = np.array(data_list) self._reset_inference_calls(policy_id) @@ -460,11 +490,12 @@ class _SimpleListCollector(_SampleCollector): return input_dict @override(_SampleCollector) - def postprocess_episode(self, - episode: MultiAgentEpisode, - is_done: bool = False, - check_dones: bool = False, - build: bool = False) -> None: + def postprocess_episode( + self, + episode: MultiAgentEpisode, + is_done: bool = False, + check_dones: bool = False, + build: bool = False) -> Union[None, SampleBatch, MultiAgentBatch]: episode_id = episode.episode_id policy_collector_group = episode.batch_builder @@ -475,7 +506,7 @@ class _SimpleListCollector(_SampleCollector): pre_batches = {} for (eps_id, agent_id), collector in self.agent_collectors.items(): # Build only if there is data and agent is part of given episode. - if collector.count == 0 or eps_id != episode_id: + if collector.agent_steps == 0 or eps_id != episode_id: continue pid = self.agent_key_to_policy_id[(eps_id, agent_id)] policy = self.policy_map[pid] @@ -517,8 +548,8 @@ class _SimpleListCollector(_SampleCollector): del other_batches[agent_id] pid = self.agent_key_to_policy_id[(episode_id, agent_id)] policy = self.policy_map[pid] - if any(pre_batch["dones"][:-1]) or len(set( - pre_batch["eps_id"])) > 1: + if any(pre_batch[SampleBatch.DONES][:-1]) or len( + set(pre_batch[SampleBatch.EPS_ID])) > 1: raise ValueError( "Batches sent to postprocessing must only contain steps " "from a single trajectory.", pre_batch) @@ -556,16 +587,19 @@ class _SimpleListCollector(_SampleCollector): post_batch, policy.view_requirements) env_steps = self.episode_steps[episode_id] - policy_collector_group.count += env_steps + policy_collector_group.env_steps += env_steps + agent_steps = self.agent_steps[episode_id] + policy_collector_group.agent_steps += agent_steps if is_done: del self.episode_steps[episode_id] + del self.agent_steps[episode_id] del self.episodes[episode_id] # Make PolicyCollectorGroup available for more agent batches in # other episodes. Do not reset count to 0. self.policy_collector_groups.append(policy_collector_group) else: - self.episode_steps[episode_id] = 0 + self.episode_steps[episode_id] = self.agent_steps[episode_id] = 0 # Build a MultiAgentBatch from the episode and return. if build: @@ -576,14 +610,15 @@ class _SimpleListCollector(_SampleCollector): ma_batch = {} for pid, collector in episode.batch_builder.policy_collectors.items(): - if collector.count > 0: + if collector.agent_steps > 0: ma_batch[pid] = collector.build() # Create the batch. ma_batch = MultiAgentBatch.wrap_as_needed( - ma_batch, env_steps=episode.batch_builder.count) + ma_batch, env_steps=episode.batch_builder.env_steps) # PolicyCollectorGroup is empty. - episode.batch_builder.count = 0 + episode.batch_builder.env_steps = 0 + episode.batch_builder.agent_steps = 0 return ma_batch @@ -592,16 +627,26 @@ class _SimpleListCollector(_SampleCollector): List[Union[MultiAgentBatch, SampleBatch]]: batches = [] # Loop through ongoing episodes and see whether their length plus - # what's already in the policy collectors reaches the fragment-len. + # what's already in the policy collectors reaches the fragment-len + # (abiding to the unit used: env-steps or agent-steps). for episode_id, episode in self.episodes.items(): - env_steps = episode.batch_builder.count + \ - self.episode_steps[episode_id] + # Measure batch size in env-steps. + if self.count_steps_by == "env_steps": + built_steps = episode.batch_builder.env_steps + ongoing_steps = self.episode_steps[episode_id] + # Measure batch-size in agent-steps. + else: + built_steps = episode.batch_builder.agent_steps + ongoing_steps = self.agent_steps[episode_id] + # Reached the fragment-len -> We should build an MA-Batch. - if env_steps >= self.rollout_fragment_length: - assert env_steps == self.rollout_fragment_length + if built_steps + ongoing_steps >= self.rollout_fragment_length: + if self.count_steps_by != "agent_steps": + assert built_steps + ongoing_steps == \ + self.rollout_fragment_length # If we reached the fragment-len only because of `episode_id` # (still ongoing) -> postprocess `episode_id` first. - if episode.batch_builder.count < self.rollout_fragment_length: + if built_steps < self.rollout_fragment_length: self.postprocess_episode(episode, is_done=False) # Build the MA-batch and return. batch = self._build_multi_agent_batch(episode=episode) diff --git a/rllib/evaluation/rollout_worker.py b/rllib/evaluation/rollout_worker.py index 31d9d6506..1579ea0b4 100644 --- a/rllib/evaluation/rollout_worker.py +++ b/rllib/evaluation/rollout_worker.py @@ -143,6 +143,7 @@ class RolloutWorker(ParallelIteratorWorker): policies_to_train: Optional[List[PolicyID]] = None, tf_session_creator: Optional[Callable[[], "tf1.Session"]] = None, rollout_fragment_length: int = 100, + count_steps_by: str = "env_steps", batch_mode: str = "truncate_episodes", episode_horizon: int = None, preprocessor_pref: str = "deepmind", @@ -177,6 +178,7 @@ class RolloutWorker(ParallelIteratorWorker): fake_sampler: bool = False, spaces: Optional[Dict[PolicyID, Tuple[gym.spaces.Space, gym.spaces.Space]]] = None, + _use_trajectory_view_api: bool = True, policy: Union[type, Dict[ str, Tuple[Optional[type], gym.Space, gym.Space, PartialTrainerConfigDict]]] = None, @@ -207,8 +209,11 @@ class RolloutWorker(ParallelIteratorWorker): tf_session_creator (Optional[Callable[[], tf1.Session]]): A function that returns a TF session. This is optional and only useful with TFPolicy. - rollout_fragment_length (int): The target number of env transitions - to include in each sample batch returned from this worker. + rollout_fragment_length (int): The target number of steps + (maesured in `count_steps_by`) to include in each sample + batch returned from this worker. + count_steps_by (str): The unit in which to count fragment + lengths. One of env_steps or agent_steps. batch_mode (str): One of the following batch modes: "truncate_episodes": Each call to sample() will return a batch of at most `rollout_fragment_length * num_envs` in size. @@ -295,6 +300,8 @@ class RolloutWorker(ParallelIteratorWorker): gym.spaces.Space]]]): An optional space dict mapping policy IDs to (obs_space, action_space)-tuples. This is used in case no Env is created on this RolloutWorker. + _use_trajectory_view_api (bool): Whether to collect samples through + the experimental Trajectory View API. policy: Obsoleted arg. Use `policy_spec` instead. """ # Deprecated arg. @@ -353,6 +360,7 @@ class RolloutWorker(ParallelIteratorWorker): raise ValueError("Policy mapping function not callable?") self.env_creator: Callable[[EnvContext], EnvType] = env_creator self.rollout_fragment_length: int = rollout_fragment_length * num_envs + self.count_steps_by: str = count_steps_by self.batch_mode: str = batch_mode self.compress_observations: bool = compress_observations self.preprocessing_enabled: bool = True @@ -459,6 +467,14 @@ class RolloutWorker(ParallelIteratorWorker): self.policy_map, self.preprocessors = self._build_policy_map( policy_dict, policy_config) + # Update Policy's view requirements from Model, only if Policy directly + # inherited from base `Policy` class. At this point here, the Policy + # must have it's Model (if any) defined and ready to output an initial + # state. + for pol in self.policy_map.values(): + if not pol._model_init_state_automatically_added: + pol._update_model_inference_view_requirements_from_init_state() + if (ray.is_initialized() and ray.worker._mode() != ray.worker.LOCAL_MODE): # Check available number of GPUs @@ -559,6 +575,7 @@ class RolloutWorker(ParallelIteratorWorker): obs_filters=self.filters, clip_rewards=clip_rewards, rollout_fragment_length=rollout_fragment_length, + count_steps_by=count_steps_by, callbacks=self.callbacks, horizon=episode_horizon, multiple_episodes_in_batch=pack, @@ -568,8 +585,8 @@ class RolloutWorker(ParallelIteratorWorker): soft_horizon=soft_horizon, no_done_at_end=no_done_at_end, observation_fn=observation_fn, - _use_trajectory_view_api=policy_config.get( - "_use_trajectory_view_api", False)) + _use_trajectory_view_api=_use_trajectory_view_api, + ) # Start the Sampler thread. self.sampler.start() else: @@ -582,6 +599,7 @@ class RolloutWorker(ParallelIteratorWorker): obs_filters=self.filters, clip_rewards=clip_rewards, rollout_fragment_length=rollout_fragment_length, + count_steps_by=count_steps_by, callbacks=self.callbacks, horizon=episode_horizon, multiple_episodes_in_batch=pack, @@ -590,8 +608,8 @@ class RolloutWorker(ParallelIteratorWorker): soft_horizon=soft_horizon, no_done_at_end=no_done_at_end, observation_fn=observation_fn, - _use_trajectory_view_api=policy_config.get( - "_use_trajectory_view_api", False)) + _use_trajectory_view_api=_use_trajectory_view_api, + ) self.input_reader: InputReader = input_creator(self.io_context) self.output_writer: OutputWriter = output_creator(self.io_context) @@ -625,7 +643,9 @@ class RolloutWorker(ParallelIteratorWorker): self.rollout_fragment_length)) batches = [self.input_reader.next()] - steps_so_far = batches[0].count + steps_so_far = batches[0].count if \ + self.count_steps_by == "env_steps" else \ + batches[0].agent_steps() # In truncate_episodes mode, never pull more than 1 batch per env. # This avoids over-running the target batch size. @@ -637,7 +657,9 @@ class RolloutWorker(ParallelIteratorWorker): while (steps_so_far < self.rollout_fragment_length and len(batches) < max_batches): batch = self.input_reader.next() - steps_so_far += batch.count + steps_so_far += batch.count if \ + self.count_steps_by == "env_steps" else \ + batch.agent_steps() batches.append(batch) batch = batches[0].concat_samples(batches) if len(batches) > 1 else \ batches[0] diff --git a/rllib/evaluation/sampler.py b/rllib/evaluation/sampler.py index fd4219366..a115a0149 100644 --- a/rllib/evaluation/sampler.py +++ b/rllib/evaluation/sampler.py @@ -129,6 +129,7 @@ class SyncSampler(SamplerInput): obs_filters: Dict[PolicyID, Filter], clip_rewards: bool, rollout_fragment_length: int, + count_steps_by: str = "env_steps", callbacks: "DefaultCallbacks", horizon: int = None, multiple_episodes_in_batch: bool = False, @@ -190,8 +191,12 @@ class SyncSampler(SamplerInput): self.perf_stats = _PerfStats() if _use_trajectory_view_api: self.sample_collector = _SimpleListCollector( - policies, clip_rewards, callbacks, multiple_episodes_in_batch, - rollout_fragment_length) + policies, + clip_rewards, + callbacks, + multiple_episodes_in_batch, + rollout_fragment_length, + count_steps_by=count_steps_by) else: self.sample_collector = None @@ -254,6 +259,7 @@ class AsyncSampler(threading.Thread, SamplerInput): obs_filters: Dict[PolicyID, Filter], clip_rewards: bool, rollout_fragment_length: int, + count_steps_by: str = "env_steps", callbacks: "DefaultCallbacks", horizon: int = None, multiple_episodes_in_batch: bool = False, @@ -282,6 +288,8 @@ class AsyncSampler(threading.Thread, SamplerInput): rollout_fragment_length (int): The length of a fragment to collect before building a SampleBatch from the data and resetting the SampleBatchBuilder object. + count_steps_by (str): Either "env_steps" or "agent_steps". + Refers to the unit of `rollout_fragment_length`. callbacks (Callbacks): The Callbacks object to use when episode events happen during rollout. horizon (Optional[int]): Hard-reset the Env @@ -336,8 +344,12 @@ class AsyncSampler(threading.Thread, SamplerInput): self._use_trajectory_view_api = _use_trajectory_view_api if _use_trajectory_view_api: self.sample_collector = _SimpleListCollector( - policies, clip_rewards, callbacks, multiple_episodes_in_batch, - rollout_fragment_length) + policies, + clip_rewards, + callbacks, + multiple_episodes_in_batch, + rollout_fragment_length, + count_steps_by=count_steps_by) else: self.sample_collector = None @@ -1046,7 +1058,6 @@ def _process_observations_w_trajectory_view_api( # Add actions, rewards, next-obs to collectors. values_dict = { "t": episode.length - 1, - "eps_id": episode.episode_id, "env_id": env_id, "agent_index": episode._agent_index(agent_id), # Action (slot 0) taken at timestep t. diff --git a/rllib/evaluation/tests/__init__.py b/rllib/evaluation/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/rllib/tests/test_rollout_worker.py b/rllib/evaluation/tests/test_rollout_worker.py similarity index 90% rename from rllib/tests/test_rollout_worker.py rename to rllib/evaluation/tests/test_rollout_worker.py index 12b92ad10..8d45f5be6 100644 --- a/rllib/tests/test_rollout_worker.py +++ b/rllib/evaluation/tests/test_rollout_worker.py @@ -1,5 +1,6 @@ from collections import Counter import gym +from gym.spaces import Box, Discrete import numpy as np import os import random @@ -13,9 +14,12 @@ from ray.rllib.env.vector_env import VectorEnv from ray.rllib.evaluation.rollout_worker import RolloutWorker from ray.rllib.evaluation.metrics import collect_metrics from ray.rllib.evaluation.postprocessing import compute_advantages +from ray.rllib.examples.env.mock_env import MockEnv, MockEnv2 +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.rllib.examples.policy.random_policy import RandomPolicy from ray.rllib.policy.policy import Policy -from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch +from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, MultiAgentBatch, \ + SampleBatch from ray.rllib.utils.annotations import override from ray.rllib.utils.test_utils import check, framework_iterator from ray.tune.registry import register_env @@ -71,39 +75,6 @@ class FailOnStepEnv(gym.Env): raise ValueError("kaboom") -class MockEnv(gym.Env): - def __init__(self, episode_length, config=None): - self.episode_length = episode_length - self.config = config - self.i = 0 - self.observation_space = gym.spaces.Discrete(1) - self.action_space = gym.spaces.Discrete(2) - - def reset(self): - self.i = 0 - return self.i - - def step(self, action): - self.i += 1 - return 0, 1, self.i >= self.episode_length, {} - - -class MockEnv2(gym.Env): - def __init__(self, episode_length): - self.episode_length = episode_length - self.i = 0 - self.observation_space = gym.spaces.Discrete(100) - self.action_space = gym.spaces.Discrete(2) - - def reset(self): - self.i = 0 - return self.i - - def step(self, action): - self.i += 1 - return self.i, 100, self.i >= self.episode_length, {} - - class MockVectorEnv(VectorEnv): def __init__(self, episode_length, num_envs): super().__init__( @@ -523,14 +494,57 @@ class TestRolloutWorker(unittest.TestCase): ev.stop() def test_truncate_episodes(self): - ev = RolloutWorker( + ev_env_steps = RolloutWorker( env_creator=lambda _: MockEnv(10), policy_spec=MockPolicy, + policy_config={"_use_trajectory_view_api": True}, rollout_fragment_length=15, batch_mode="truncate_episodes") - batch = ev.sample() + batch = ev_env_steps.sample() self.assertEqual(batch.count, 15) - ev.stop() + self.assertTrue(isinstance(batch, SampleBatch)) + ev_env_steps.stop() + + action_space = Discrete(2) + obs_space = Box(float("-inf"), float("inf"), (4, ), dtype=np.float32) + ev_agent_steps = RolloutWorker( + env_creator=lambda _: MultiAgentCartPole({"num_agents": 4}), + policy_spec={ + "pol0": (MockPolicy, obs_space, action_space, {}), + "pol1": (MockPolicy, obs_space, action_space, {}), + }, + policy_config={"_use_trajectory_view_api": True}, + policy_mapping_fn=lambda ag: "pol0" if ag == 0 else "pol1", + rollout_fragment_length=301, + count_steps_by="env_steps", + batch_mode="truncate_episodes", + ) + batch = ev_agent_steps.sample() + self.assertTrue(isinstance(batch, MultiAgentBatch)) + self.assertGreater(batch.agent_steps(), 301) + self.assertEqual(batch.env_steps(), 301) + ev_agent_steps.stop() + + ev_agent_steps = RolloutWorker( + env_creator=lambda _: MultiAgentCartPole({"num_agents": 4}), + policy_spec={ + "pol0": (MockPolicy, obs_space, action_space, {}), + "pol1": (MockPolicy, obs_space, action_space, {}), + }, + policy_config={"_use_trajectory_view_api": True}, + policy_mapping_fn=lambda ag: "pol0" if ag == 0 else "pol1", + rollout_fragment_length=301, + count_steps_by="agent_steps", + batch_mode="truncate_episodes") + batch = ev_agent_steps.sample() + self.assertTrue(isinstance(batch, MultiAgentBatch)) + self.assertLess(batch.env_steps(), 301) + # When counting agent steps, the count may be slightly larger than + # rollout_fragment_length, b/c we have up to N agents stepping in each + # env step and we only check, whether we should build after each env + # step. + self.assertGreaterEqual(batch.agent_steps(), 301) + ev_agent_steps.stop() def test_complete_episodes(self): ev = RolloutWorker( diff --git a/rllib/evaluation/tests/test_trajectory_view_api.py b/rllib/evaluation/tests/test_trajectory_view_api.py index 3fc23289c..a50978bfd 100644 --- a/rllib/evaluation/tests/test_trajectory_view_api.py +++ b/rllib/evaluation/tests/test_trajectory_view_api.py @@ -1,13 +1,16 @@ import copy import gym from gym.spaces import Box, Discrete +import numpy as np import time import unittest import ray +from ray import tune import ray.rllib.agents.dqn as dqn import ray.rllib.agents.ppo as ppo from ray.rllib.examples.env.debug_counter_env import MultiAgentDebugCounterEnv +from ray.rllib.examples.env.multi_agent import MultiAgentCartPole from ray.rllib.evaluation.rollout_worker import RolloutWorker from ray.rllib.examples.policy.episode_env_aware_policy import \ EpisodeEnvAwareLSTMPolicy @@ -59,7 +62,7 @@ class TestTrajectoryViewAPI(unittest.TestCase): assert view_req_policy[key].data_col is None else: assert view_req_policy[key].data_col == SampleBatch.OBS - assert view_req_policy[key].data_rel_pos == 1 + assert view_req_policy[key].shift == 1 rollout_worker = trainer.workers.local_worker() sample_batch = rollout_worker.sample() expected_count = \ @@ -99,10 +102,10 @@ class TestTrajectoryViewAPI(unittest.TestCase): if key == SampleBatch.PREV_ACTIONS: assert view_req_policy[key].data_col == SampleBatch.ACTIONS - assert view_req_policy[key].data_rel_pos == -1 + assert view_req_policy[key].shift == -1 elif key == SampleBatch.PREV_REWARDS: assert view_req_policy[key].data_col == SampleBatch.REWARDS - assert view_req_policy[key].data_rel_pos == -1 + assert view_req_policy[key].shift == -1 elif key not in [ SampleBatch.NEXT_OBS, SampleBatch.PREV_ACTIONS, SampleBatch.PREV_REWARDS @@ -110,7 +113,7 @@ class TestTrajectoryViewAPI(unittest.TestCase): assert view_req_policy[key].data_col is None else: assert view_req_policy[key].data_col == SampleBatch.OBS - assert view_req_policy[key].data_rel_pos == 1 + assert view_req_policy[key].shift == 1 trainer.stop() def test_traj_view_simple_performance(self): @@ -295,6 +298,38 @@ class TestTrajectoryViewAPI(unittest.TestCase): pol_batch_wo = result.policy_batches["pol0"] check(pol_batch_w.data, pol_batch_wo.data) + def test_counting_by_agent_steps(self): + """Test whether a PPOTrainer can be built with all frameworks.""" + config = copy.deepcopy(ppo.DEFAULT_CONFIG) + action_space = Discrete(2) + obs_space = Box(float("-inf"), float("inf"), (4, ), dtype=np.float32) + + config["num_workers"] = 2 + config["num_sgd_iter"] = 2 + config["framework"] = "torch" + config["rollout_fragment_length"] = 21 + config["train_batch_size"] = 147 + config["multiagent"] = { + "policies": { + "p0": (None, obs_space, action_space, {}), + "p1": (None, obs_space, action_space, {}), + }, + "policy_mapping_fn": lambda aid: "p{}".format(aid), + "count_steps_by": "agent_steps", + } + tune.register_env( + "ma_cartpole", lambda _: MultiAgentCartPole({"num_agents": 2})) + num_iterations = 2 + trainer = ppo.PPOTrainer(config=config, env="ma_cartpole") + results = None + for i in range(num_iterations): + results = trainer.train() + self.assertGreater(results["timesteps_total"], + num_iterations * config["train_batch_size"]) + self.assertLess(results["timesteps_total"], + (num_iterations + 1) * config["train_batch_size"]) + trainer.stop() + def analyze_rnn_batch(batch, max_seq_len): count = batch.count diff --git a/rllib/evaluation/worker_set.py b/rllib/evaluation/worker_set.py index 9172e5f4a..80cf617bb 100644 --- a/rllib/evaluation/worker_set.py +++ b/rllib/evaluation/worker_set.py @@ -74,12 +74,15 @@ class WorkerSet: self.add_workers(num_workers) # If num_workers > 0, get the action_spaces and observation_spaces - # to not be forced to create an Env on the driver. + # to not be forced to create an Env on the local worker. if self._remote_workers: remote_spaces = ray.get(self.remote_workers( )[0].foreach_policy.remote( lambda p, pid: (pid, p.observation_space, p.action_space))) - spaces = {e[0]: (e[1], e[2]) for e in remote_spaces} + spaces = { + e[0]: (getattr(e[1], "original_space", e[1]), e[2]) + for e in remote_spaces + } else: spaces = None @@ -260,13 +263,13 @@ class WorkerSet: elif config["input"] == "sampler": input_creator = (lambda ioctx: ioctx.default_sampler_input()) elif isinstance(config["input"], dict): - input_creator = (lambda ioctx: ShuffledInput( - MixedInput(config["input"], ioctx), config[ - "shuffle_buffer_size"])) + input_creator = ( + lambda ioctx: ShuffledInput(MixedInput(config["input"], ioctx), + config["shuffle_buffer_size"])) else: - input_creator = (lambda ioctx: ShuffledInput( - JsonReader(config["input"], ioctx), config[ - "shuffle_buffer_size"])) + input_creator = ( + lambda ioctx: ShuffledInput(JsonReader(config["input"], ioctx), + config["shuffle_buffer_size"])) if isinstance(config["output"], FunctionType): output_creator = config["output"] @@ -321,6 +324,7 @@ class WorkerSet: tf_session_creator=(session_creator if config["tf_session_args"] else None), rollout_fragment_length=config["rollout_fragment_length"], + count_steps_by=config["multiagent"]["count_steps_by"], batch_mode=config["batch_mode"], episode_horizon=config["horizon"], preprocessor_pref=config["preprocessor_pref"], @@ -352,6 +356,7 @@ class WorkerSet: fake_sampler=config["fake_sampler"], extra_python_environs=extra_python_environs, spaces=spaces, + _use_trajectory_view_api=config["_use_trajectory_view_api"], ) return worker diff --git a/rllib/examples/env/mock_env.py b/rllib/examples/env/mock_env.py new file mode 100644 index 000000000..8ddbb9b69 --- /dev/null +++ b/rllib/examples/env/mock_env.py @@ -0,0 +1,46 @@ +import gym + + +class MockEnv(gym.Env): + """Mock environment for testing purposes. + + Observation=0, reward=1.0, episode-len is configurable. + Actions are ignored. + """ + + def __init__(self, episode_length, config=None): + self.episode_length = episode_length + self.config = config + self.i = 0 + self.observation_space = gym.spaces.Discrete(1) + self.action_space = gym.spaces.Discrete(2) + + def reset(self): + self.i = 0 + return 0 + + def step(self, action): + self.i += 1 + return 0, 1.0, self.i >= self.episode_length, {} + + +class MockEnv2(gym.Env): + """Mock environment for testing purposes. + + Observation=ts (discrete space!), reward=100.0, episode-len is + configurable. Actions are ignored. + """ + + def __init__(self, episode_length): + self.episode_length = episode_length + self.i = 0 + self.observation_space = gym.spaces.Discrete(100) + self.action_space = gym.spaces.Discrete(2) + + def reset(self): + self.i = 0 + return self.i + + def step(self, action): + self.i += 1 + return self.i, 100.0, self.i >= self.episode_length, {} diff --git a/rllib/examples/env/multi_agent.py b/rllib/examples/env/multi_agent.py index 5d4ffe863..096dea205 100644 --- a/rllib/examples/env/multi_agent.py +++ b/rllib/examples/env/multi_agent.py @@ -1,8 +1,8 @@ import gym from ray.rllib.env.multi_agent_env import MultiAgentEnv +from ray.rllib.examples.env.mock_env import MockEnv, MockEnv2 from ray.rllib.examples.env.stateless_cartpole import StatelessCartPole -from ray.rllib.tests.test_rollout_worker import MockEnv, MockEnv2 def make_multiagent(env_name_or_creator): diff --git a/rllib/examples/policy/episode_env_aware_policy.py b/rllib/examples/policy/episode_env_aware_policy.py index 89d4c525e..e632e1694 100644 --- a/rllib/examples/policy/episode_env_aware_policy.py +++ b/rllib/examples/policy/episode_env_aware_policy.py @@ -28,15 +28,15 @@ class EpisodeEnvAwareLSTMPolicy(RandomPolicy): "t": ViewRequirement(), SampleBatch.OBS: ViewRequirement(), SampleBatch.PREV_ACTIONS: ViewRequirement( - SampleBatch.ACTIONS, space=self.action_space, data_rel_pos=-1), + SampleBatch.ACTIONS, space=self.action_space, shift=-1), SampleBatch.PREV_REWARDS: ViewRequirement( - SampleBatch.REWARDS, data_rel_pos=-1), + SampleBatch.REWARDS, shift=-1), } for i in range(2): self.model.inference_view_requirements["state_in_{}".format(i)] = \ ViewRequirement( "state_out_{}".format(i), - data_rel_pos=-1, + shift=-1, space=self.state_space) self.model.inference_view_requirements[ "state_out_{}".format(i)] = \ @@ -45,7 +45,7 @@ class EpisodeEnvAwareLSTMPolicy(RandomPolicy): self.view_requirements = dict( **{ SampleBatch.NEXT_OBS: ViewRequirement( - SampleBatch.OBS, data_rel_pos=1), + SampleBatch.OBS, shift=1), SampleBatch.ACTIONS: ViewRequirement(space=self.action_space), SampleBatch.REWARDS: ViewRequirement(), SampleBatch.DONES: ViewRequirement(), @@ -106,7 +106,7 @@ class EpisodeEnvAwareAttentionPolicy(RandomPolicy): "state_in_0": ViewRequirement( "state_out_0", # Provide state outs -50 to -1 as "state-in". - data_rel_pos="-50:-1", + shift="-50:-1", # Repeat the incoming state every n time steps (usually max seq # len). batch_repeat_value=self.config["model"]["max_seq_len"], diff --git a/rllib/examples/policy/rock_paper_scissors_dummies.py b/rllib/examples/policy/rock_paper_scissors_dummies.py index 011d49e56..72b2fdd51 100644 --- a/rllib/examples/policy/rock_paper_scissors_dummies.py +++ b/rllib/examples/policy/rock_paper_scissors_dummies.py @@ -16,7 +16,7 @@ class AlwaysSameHeuristic(Policy): self.view_requirements.update({ "state_in_0": ViewRequirement( "state_out_0", - data_rel_pos=-1, + shift=-1, space=gym.spaces.Box(0, 100, shape=(), dtype=np.int32)) }) diff --git a/rllib/examples/two_trainer_workflow.py b/rllib/examples/two_trainer_workflow.py index e0f828a7b..a87a92405 100644 --- a/rllib/examples/two_trainer_workflow.py +++ b/rllib/examples/two_trainer_workflow.py @@ -81,7 +81,8 @@ def custom_training_workflow(workers: WorkerSet, config: dict): # PPO sub-flow. ppo_train_op = r2.for_each(SelectExperiences(["ppo_policy"])) \ - .combine(ConcatBatches(min_batch_size=200)) \ + .combine(ConcatBatches( + min_batch_size=200, count_steps_by="env_steps")) \ .for_each(add_ppo_metrics) \ .for_each(StandardizeFields(["advantages"])) \ .for_each(TrainOneStep( @@ -143,6 +144,7 @@ if __name__ == "__main__": # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0. "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")), "framework": "torch" if args.torch else "tf", + "_use_trajectory_view_api": True, } stop = { diff --git a/rllib/execution/rollout_ops.py b/rllib/execution/rollout_ops.py index 818254c2d..baaa26357 100644 --- a/rllib/execution/rollout_ops.py +++ b/rllib/execution/rollout_ops.py @@ -141,13 +141,15 @@ class ConcatBatches: Examples: >>> rollouts = ParallelRollouts(...) - >>> rollouts = rollouts.combine(ConcatBatches(min_batch_size=10000)) + >>> rollouts = rollouts.combine(ConcatBatches( + ... min_batch_size=10000, count_steps_by="env_steps")) >>> print(next(rollouts).count) 10000 """ - def __init__(self, min_batch_size: int): + def __init__(self, min_batch_size: int, count_steps_by: str = "env_steps"): self.min_batch_size = min_batch_size + self.count_steps_by = count_steps_by self.buffer = [] self.count = 0 self.batch_start_time = None @@ -159,7 +161,15 @@ class ConcatBatches: def __call__(self, batch: SampleBatchType) -> List[SampleBatchType]: _check_sample_batch_type(batch) self.buffer.append(batch) - self.count += batch.count + + if self.count_steps_by == "env_steps": + self.count += batch.count + else: + assert isinstance(batch, MultiAgentBatch), \ + "`count_steps_by=agent_steps` only allowed in multi-agent " \ + "environments!" + self.count += batch.agent_steps() + if self.count >= self.min_batch_size: if self.count > self.min_batch_size * 2: logger.info("Collected more training samples than expected " diff --git a/rllib/execution/tree_agg.py b/rllib/execution/tree_agg.py index 344a22e20..69e06a4b0 100644 --- a/rllib/execution/tree_agg.py +++ b/rllib/execution/tree_agg.py @@ -51,7 +51,9 @@ class Aggregator(ParallelIteratorWorker): .flatten() \ .combine( ConcatBatches( - min_batch_size=config["train_batch_size"])) + min_batch_size=config["train_batch_size"], + count_steps_by=config["multiagent"]["count_steps_by"], + )) for train_batch in it: yield train_batch diff --git a/rllib/models/modelv2.py b/rllib/models/modelv2.py index 21bc139d4..38478857c 100644 --- a/rllib/models/modelv2.py +++ b/rllib/models/modelv2.py @@ -61,8 +61,7 @@ class ModelV2: self.time_major = self.model_config.get("_time_major") # Basic view requirement for all models: Use the observation as input. self.inference_view_requirements = { - SampleBatch.OBS: ViewRequirement( - data_rel_pos=0, space=self.obs_space), + SampleBatch.OBS: ViewRequirement(shift=0, space=self.obs_space), } # TODO: (sven): Get rid of `get_initial_state` once Trajectory @@ -315,6 +314,29 @@ class ModelV2: """ return self.time_major is True + # TODO: (sven) Experimental method. + def get_input_dict(self, sample_batch, + index: int = -1) -> Dict[str, TensorType]: + if index < 0: + index = sample_batch.count - 1 + + input_dict = {} + for view_col, view_req in self.inference_view_requirements.items(): + # Create batches of size 1 (single-agent input-dict). + + # Index range. + if isinstance(index, tuple): + data = sample_batch[view_col][index[0]:index[1] + 1] + input_dict[view_col] = np.array([data]) + # Single index. + else: + input_dict[view_col] = sample_batch[view_col][index:index + 1] + + # Add valid `seq_lens`, just in case RNNs need it. + input_dict["seq_lens"] = np.array([1], dtype=np.int32) + + return input_dict + class NullContextManager: """No-op context manager""" diff --git a/rllib/models/preprocessors.py b/rllib/models/preprocessors.py index 51a6f5442..c1b7803c4 100644 --- a/rllib/models/preprocessors.py +++ b/rllib/models/preprocessors.py @@ -80,9 +80,9 @@ class Preprocessor: obs_space = gym.spaces.Box(-1., 1., self.shape, dtype=np.float32) # Stash the unwrapped space so that we can unwrap dict and tuple spaces # automatically in model.py - if (isinstance(self, TupleFlatteningPreprocessor) - or isinstance(self, DictFlatteningPreprocessor) - or isinstance(self, RepeatedValuesPreprocessor)): + classes = (DictFlatteningPreprocessor, OneHotPreprocessor, + RepeatedValuesPreprocessor, TupleFlatteningPreprocessor) + if isinstance(self, classes): obs_space.original_space = self._obs_space return obs_space diff --git a/rllib/models/tf/recurrent_net.py b/rllib/models/tf/recurrent_net.py index 57ecb22f0..f939c7ae3 100644 --- a/rllib/models/tf/recurrent_net.py +++ b/rllib/models/tf/recurrent_net.py @@ -178,10 +178,10 @@ class LSTMWrapper(RecurrentNetwork): if model_config["lstm_use_prev_action"]: self.inference_view_requirements[SampleBatch.PREV_ACTIONS] = \ ViewRequirement(SampleBatch.ACTIONS, space=self.action_space, - data_rel_pos=-1) + shift=-1) if model_config["lstm_use_prev_reward"]: self.inference_view_requirements[SampleBatch.PREV_REWARDS] = \ - ViewRequirement(SampleBatch.REWARDS, data_rel_pos=-1) + ViewRequirement(SampleBatch.REWARDS, shift=-1) @override(RecurrentNetwork) def forward(self, input_dict: Dict[str, TensorType], diff --git a/rllib/models/torch/recurrent_net.py b/rllib/models/torch/recurrent_net.py index 8e0d2263c..d558bf3db 100644 --- a/rllib/models/torch/recurrent_net.py +++ b/rllib/models/torch/recurrent_net.py @@ -159,10 +159,10 @@ class LSTMWrapper(RecurrentNetwork, nn.Module): if model_config["lstm_use_prev_action"]: self.inference_view_requirements[SampleBatch.PREV_ACTIONS] = \ ViewRequirement(SampleBatch.ACTIONS, space=self.action_space, - data_rel_pos=-1) + shift=-1) if model_config["lstm_use_prev_reward"]: self.inference_view_requirements[SampleBatch.PREV_REWARDS] = \ - ViewRequirement(SampleBatch.REWARDS, data_rel_pos=-1) + ViewRequirement(SampleBatch.REWARDS, shift=-1) @override(RecurrentNetwork) def forward(self, input_dict: Dict[str, TensorType], diff --git a/rllib/offline/is_estimator.py b/rllib/offline/is_estimator.py index 1591be84a..619cc0dee 100644 --- a/rllib/offline/is_estimator.py +++ b/rllib/offline/is_estimator.py @@ -1,40 +1,40 @@ -from ray.rllib.offline.off_policy_estimator import OffPolicyEstimator, \ - OffPolicyEstimate -from ray.rllib.utils.annotations import override -from ray.rllib.utils.typing import SampleBatchType - - -class ImportanceSamplingEstimator(OffPolicyEstimator): - """The step-wise IS estimator. - - Step-wise IS estimator described in https://arxiv.org/pdf/1511.03722.pdf""" - - @override(OffPolicyEstimator) - def estimate(self, batch: SampleBatchType) -> OffPolicyEstimate: - self.check_can_estimate_for(batch) - - rewards, old_prob = batch["rewards"], batch["action_prob"] - new_prob = self.action_prob(batch) - - # calculate importance ratios - p = [] - for t in range(batch.count - 1): - if t == 0: - pt_prev = 1.0 - else: - pt_prev = p[t - 1] - p.append(pt_prev * new_prob[t] / old_prob[t]) - - # calculate stepwise IS estimate - V_prev, V_step_IS = 0.0, 0.0 - for t in range(batch.count - 1): - V_prev += rewards[t] * self.gamma**t - V_step_IS += p[t] * rewards[t] * self.gamma**t - - estimation = OffPolicyEstimate( - "is", { - "V_prev": V_prev, - "V_step_IS": V_step_IS, - "V_gain_est": V_step_IS / max(1e-8, V_prev), - }) - return estimation +from ray.rllib.offline.off_policy_estimator import OffPolicyEstimator, \ + OffPolicyEstimate +from ray.rllib.utils.annotations import override +from ray.rllib.utils.typing import SampleBatchType + + +class ImportanceSamplingEstimator(OffPolicyEstimator): + """The step-wise IS estimator. + + Step-wise IS estimator described in https://arxiv.org/pdf/1511.03722.pdf""" + + @override(OffPolicyEstimator) + def estimate(self, batch: SampleBatchType) -> OffPolicyEstimate: + self.check_can_estimate_for(batch) + + rewards, old_prob = batch["rewards"], batch["action_prob"] + new_prob = self.action_prob(batch) + + # calculate importance ratios + p = [] + for t in range(batch.count): + if t == 0: + pt_prev = 1.0 + else: + pt_prev = p[t - 1] + p.append(pt_prev * new_prob[t] / old_prob[t]) + + # calculate stepwise IS estimate + V_prev, V_step_IS = 0.0, 0.0 + for t in range(batch.count): + V_prev += rewards[t] * self.gamma**t + V_step_IS += p[t] * rewards[t] * self.gamma**t + + estimation = OffPolicyEstimate( + "is", { + "V_prev": V_prev, + "V_step_IS": V_step_IS, + "V_gain_est": V_step_IS / max(1e-8, V_prev), + }) + return estimation diff --git a/rllib/offline/wis_estimator.py b/rllib/offline/wis_estimator.py index a99d6643f..74eb342a4 100644 --- a/rllib/offline/wis_estimator.py +++ b/rllib/offline/wis_estimator.py @@ -1,54 +1,54 @@ -from ray.rllib.offline.off_policy_estimator import OffPolicyEstimator, \ - OffPolicyEstimate -from ray.rllib.policy import Policy -from ray.rllib.utils.annotations import override -from ray.rllib.utils.typing import SampleBatchType - - -class WeightedImportanceSamplingEstimator(OffPolicyEstimator): - """The weighted step-wise IS estimator. - - Step-wise WIS estimator in https://arxiv.org/pdf/1511.03722.pdf""" - - def __init__(self, policy: Policy, gamma: float): - super().__init__(policy, gamma) - self.filter_values = [] - self.filter_counts = [] - - @override(OffPolicyEstimator) - def estimate(self, batch: SampleBatchType) -> OffPolicyEstimate: - self.check_can_estimate_for(batch) - - rewards, old_prob = batch["rewards"], batch["action_prob"] - new_prob = self.action_prob(batch) - - # calculate importance ratios - p = [] - for t in range(batch.count - 1): - if t == 0: - pt_prev = 1.0 - else: - pt_prev = p[t - 1] - p.append(pt_prev * new_prob[t] / old_prob[t]) - for t, v in enumerate(p): - if t >= len(self.filter_values): - self.filter_values.append(v) - self.filter_counts.append(1.0) - else: - self.filter_values[t] += v - self.filter_counts[t] += 1.0 - - # calculate stepwise weighted IS estimate - V_prev, V_step_WIS = 0.0, 0.0 - for t in range(batch.count - 1): - V_prev += rewards[t] * self.gamma**t - w_t = self.filter_values[t] / self.filter_counts[t] - V_step_WIS += p[t] / w_t * rewards[t] * self.gamma**t - - estimation = OffPolicyEstimate( - "wis", { - "V_prev": V_prev, - "V_step_WIS": V_step_WIS, - "V_gain_est": V_step_WIS / max(1e-8, V_prev), - }) - return estimation +from ray.rllib.offline.off_policy_estimator import OffPolicyEstimator, \ + OffPolicyEstimate +from ray.rllib.policy import Policy +from ray.rllib.utils.annotations import override +from ray.rllib.utils.typing import SampleBatchType + + +class WeightedImportanceSamplingEstimator(OffPolicyEstimator): + """The weighted step-wise IS estimator. + + Step-wise WIS estimator in https://arxiv.org/pdf/1511.03722.pdf""" + + def __init__(self, policy: Policy, gamma: float): + super().__init__(policy, gamma) + self.filter_values = [] + self.filter_counts = [] + + @override(OffPolicyEstimator) + def estimate(self, batch: SampleBatchType) -> OffPolicyEstimate: + self.check_can_estimate_for(batch) + + rewards, old_prob = batch["rewards"], batch["action_prob"] + new_prob = self.action_prob(batch) + + # calculate importance ratios + p = [] + for t in range(batch.count): + if t == 0: + pt_prev = 1.0 + else: + pt_prev = p[t - 1] + p.append(pt_prev * new_prob[t] / old_prob[t]) + for t, v in enumerate(p): + if t >= len(self.filter_values): + self.filter_values.append(v) + self.filter_counts.append(1.0) + else: + self.filter_values[t] += v + self.filter_counts[t] += 1.0 + + # calculate stepwise weighted IS estimate + V_prev, V_step_WIS = 0.0, 0.0 + for t in range(batch.count): + V_prev += rewards[t] * self.gamma**t + w_t = self.filter_values[t] / self.filter_counts[t] + V_step_WIS += p[t] / w_t * rewards[t] * self.gamma**t + + estimation = OffPolicyEstimate( + "wis", { + "V_prev": V_prev, + "V_step_WIS": V_step_WIS, + "V_gain_est": V_step_WIS / max(1e-8, V_prev), + }) + return estimation diff --git a/rllib/policy/dynamic_tf_policy.py b/rllib/policy/dynamic_tf_policy.py index 900d6cb57..432e384f2 100644 --- a/rllib/policy/dynamic_tf_policy.py +++ b/rllib/policy/dynamic_tf_policy.py @@ -80,8 +80,6 @@ class DynamicTFPolicy(TFPolicy): ], Tuple[TensorType, type, List[TensorType]]]] = None, existing_inputs: Optional[Dict[str, "tf1.placeholder"]] = None, existing_model: Optional[ModelV2] = None, - view_requirements_fn: Optional[Callable[[Policy], Dict[ - str, ViewRequirement]]] = None, get_batch_divisibility_req: Optional[Callable[[Policy], int]] = None, obs_include_prev_action_reward: bool = True): @@ -235,7 +233,8 @@ class DynamicTFPolicy(TFPolicy): tf.float32, [None], name="prev_reward"), }) # Placeholder for (sampling steps) timestep (int). - timestep = tf1.placeholder(tf.int64, (), name="timestep") + timestep = tf1.placeholder_with_default( + tf.zeros((), dtype=tf.int64), (), name="timestep") # Placeholder for `is_exploring` flag. explore = tf1.placeholder_with_default( True, (), name="is_exploring") @@ -292,14 +291,6 @@ class DynamicTFPolicy(TFPolicy): action_distribution=action_dist, timestep=timestep, explore=explore) - if self.config["_use_trajectory_view_api"]: - self._dummy_batch[SampleBatch.ACTION_DIST_INPUTS] = \ - np.zeros( - [1 if not s else s for s in - dist_inputs.shape.as_list()]) - self._input_dict[SampleBatch.ACTION_DIST_INPUTS] = \ - tf1.placeholder(shape=dist_inputs.shape.as_list(), - dtype=tf.float32) # Phase 1 init. sess = tf1.get_default_session() or tf1.Session() @@ -417,42 +408,37 @@ class DynamicTFPolicy(TFPolicy): input_dict/dummy_batch tuple. """ input_dict = {} - dummy_batch = {} for view_col, view_req in view_requirements.items(): # Point state_in to the already existing self._state_inputs. mo = re.match("state_in_(\d+)", view_col) if mo is not None: input_dict[view_col] = self._state_inputs[int(mo.group(1))] - dummy_batch[view_col] = np.zeros_like( - [view_req.space.sample()]) # State-outs (no placeholders needed). elif view_col.startswith("state_out_"): - dummy_batch[view_col] = np.zeros_like( - [view_req.space.sample()]) + continue # Skip action dist inputs placeholder (do later). elif view_col == SampleBatch.ACTION_DIST_INPUTS: continue elif view_col in existing_inputs: input_dict[view_col] = existing_inputs[view_col] - dummy_batch[view_col] = np.zeros( - shape=[ - 1 if s is None else s - for s in existing_inputs[view_col].shape.as_list() - ], - dtype=existing_inputs[view_col].dtype.as_numpy_dtype) # All others. else: if view_req.used_for_training: input_dict[view_col] = get_placeholder( space=view_req.space, name=view_col) - dummy_batch[view_col] = np.zeros_like( - [view_req.space.sample()]) + dummy_batch = self._get_dummy_batch_from_view_requirements( + batch_size=32) + return input_dict, dummy_batch def _initialize_loss_from_dummy_batch( self, auto_remove_unneeded_view_reqs: bool = True, stats_fn=None) -> None: + # Create the optimizer/exploration optimizer here. Some initialization + # steps (e.g. exploration postprocessing) may need this. + self._optimizer = self.optimizer() + # Test calls depend on variable init, so initialize model first. self._sess.run(tf1.global_variables_initializer()) @@ -509,6 +495,8 @@ class DynamicTFPolicy(TFPolicy): batch_for_postproc = UsageTrackingDict(sb) batch_for_postproc.count = sb.count logger.info("Testing `postprocess_trajectory` w/ dummy batch.") + self.exploration.postprocess_trajectory(self, batch_for_postproc, + self._sess) postprocessed_batch = self.postprocess_trajectory(batch_for_postproc) # Add new columns automatically to (loss) input_dict. if self.config["_use_trajectory_view_api"]: @@ -588,7 +576,8 @@ class DynamicTFPolicy(TFPolicy): batch_for_postproc.accessed_keys # Tag those only needed for post-processing. for key in batch_for_postproc.accessed_keys: - if key not in train_batch.accessed_keys: + if key not in train_batch.accessed_keys and \ + key not in self.model.inference_view_requirements: self.view_requirements[key].used_for_training = False if key in self._loss_input_dict: del self._loss_input_dict[key] diff --git a/rllib/policy/eager_tf_policy.py b/rllib/policy/eager_tf_policy.py index 62dbe3148..758cfc948 100644 --- a/rllib/policy/eager_tf_policy.py +++ b/rllib/policy/eager_tf_policy.py @@ -194,7 +194,6 @@ def build_eager_tf_policy(name, action_sampler_fn=None, action_distribution_fn=None, mixins=None, - view_requirements_fn=None, obs_include_prev_action_reward=True, get_batch_divisibility_req=None): """Build an eager TF policy. @@ -265,9 +264,6 @@ def build_eager_tf_policy(name, for s in self.model.get_initial_state() ] - # Update this Policy's ViewRequirements (if function given). - if callable(view_requirements_fn): - self.view_requirements.update(view_requirements_fn(self)) # Combine view_requirements for Model and Policy. self.view_requirements.update( self.model.inference_view_requirements) @@ -275,12 +271,6 @@ def build_eager_tf_policy(name, if before_loss_init: before_loss_init(self, observation_space, action_space, config) - self._initialize_loss_from_dummy_batch( - auto_remove_unneeded_view_reqs=True, - stats_fn=stats_fn, - ) - self._loss_initialized = True - if optimizer_fn: optimizers = optimizer_fn(self, config) else: @@ -293,10 +283,16 @@ def build_eager_tf_policy(name, # Just like torch Policy does. self._optimizer = optimizers[0] if optimizers else None + self._initialize_loss_from_dummy_batch( + auto_remove_unneeded_view_reqs=True, + stats_fn=stats_fn, + ) + self._loss_initialized = True + if after_init: after_init(self, observation_space, action_space, config) - # Got to reset global_timestep again after this fake run-through. + # Got to reset global_timestep again after fake run-throughs. self.global_timestep = 0 @override(Policy) @@ -397,7 +393,7 @@ def build_eager_tf_policy(name, if action_sampler_fn: dist_inputs = None state_out = [] - actions, logp = self.action_sampler_fn( + actions, logp = action_sampler_fn( self, self.model, input_dict[SampleBatch.CUR_OBS], @@ -410,7 +406,7 @@ def build_eager_tf_policy(name, timestep=timestep, explore=explore) if action_distribution_fn: - dist_inputs, dist_class, state_out = \ + dist_inputs, self.dist_class, state_out = \ action_distribution_fn( self, self.model, input_dict[SampleBatch.CUR_OBS], @@ -418,11 +414,10 @@ def build_eager_tf_policy(name, timestep=timestep, is_training=False) else: - dist_class = self.dist_class dist_inputs, state_out = self.model( input_dict, state_batches, seq_lens) - action_dist = dist_class(dist_inputs, self.model) + action_dist = self.dist_class(dist_inputs, self.model) # Get the exploration action from the forward results. actions, logp = self.exploration.get_exploration_action( @@ -466,12 +461,12 @@ def build_eager_tf_policy(name, "is_training": tf.constant(False), } if obs_include_prev_action_reward: - input_dict.update({ - SampleBatch.PREV_ACTIONS: tf.convert_to_tensor( - prev_action_batch), - SampleBatch.PREV_REWARDS: tf.convert_to_tensor( - prev_reward_batch), - }) + if prev_action_batch is not None: + input_dict[SampleBatch.PREV_ACTIONS] = \ + tf.convert_to_tensor(prev_action_batch) + if prev_reward_batch is not None: + input_dict[SampleBatch.PREV_REWARDS] = \ + tf.convert_to_tensor(prev_reward_batch) # Exploration hook before each forward pass. self.exploration.before_compute_actions(explore=False) @@ -559,7 +554,9 @@ def build_eager_tf_policy(name, @override(Policy) def get_initial_state(self): - return self.model.get_initial_state() + if hasattr(self, "model"): + return self.model.get_initial_state() + return [] def get_session(self): return None # None implies eager @@ -597,14 +594,6 @@ def build_eager_tf_policy(name, self._is_training = True with tf.GradientTape(persistent=gradients_fn is not None) as tape: - # TODO: set seq len and state-in properly - state_in = [] - for i in range(self.num_state_tensors()): - state_in.append(samples["state_in_{}".format(i)]) - self._state_in = state_in - - model_out, _ = self.model(samples, self._state_in, - samples.get("seq_lens")) loss = loss_fn(self, self.model, self.dist_class, samples) variables = self.model.trainable_variables() diff --git a/rllib/policy/policy.py b/rllib/policy/policy.py index a98c50f46..a1e92ac37 100644 --- a/rllib/policy/policy.py +++ b/rllib/policy/policy.py @@ -92,6 +92,7 @@ class Policy(metaclass=ABCMeta): self.view_requirements = view_reqs else: self.view_requirements.update(view_reqs) + self._model_init_state_automatically_added = False @abstractmethod @DeveloperAPI @@ -278,7 +279,8 @@ class Policy(metaclass=ABCMeta): # `self.compute_actions()`. state_batches = [ # TODO: (sven) remove unsqueezing code here for non-traj.view API. - s if self.config["_use_trajectory_view_api"] else s.unsqueeze(0) + s if self.config.get("_use_trajectory_view_api", False) else + s.unsqueeze(0) if torch and isinstance(s, torch.Tensor) else np.expand_dims(s, 0) for k, s in input_dict.items() if k[:9] == "state_in_" ] @@ -564,16 +566,25 @@ class Policy(metaclass=ABCMeta): SampleBatch.OBS: ViewRequirement(space=self.observation_space), SampleBatch.NEXT_OBS: ViewRequirement( data_col=SampleBatch.OBS, - data_rel_pos=1, + shift=1, space=self.observation_space), SampleBatch.ACTIONS: ViewRequirement(space=self.action_space), + # For backward compatibility with custom Models that don't specify + # these explicitly (will be removed by Policy if not used). + SampleBatch.PREV_ACTIONS: ViewRequirement( + data_col=SampleBatch.ACTIONS, + shift=-1, + space=self.action_space), SampleBatch.REWARDS: ViewRequirement(), + # For backward compatibility with custom Models that don't specify + # these explicitly (will be removed by Policy if not used). + SampleBatch.PREV_REWARDS: ViewRequirement( + data_col=SampleBatch.REWARDS, shift=-1), SampleBatch.DONES: ViewRequirement(), SampleBatch.INFOS: ViewRequirement(), SampleBatch.EPS_ID: ViewRequirement(), SampleBatch.UNROLL_ID: ViewRequirement(), SampleBatch.AGENT_INDEX: ViewRequirement(), - SampleBatch.UNROLL_ID: ViewRequirement(), "t": ViewRequirement(), } @@ -616,6 +627,7 @@ class Policy(metaclass=ABCMeta): -1.0, 1.0, shape=value.shape[1:], dtype=value.dtype)) batch_for_postproc = UsageTrackingDict(self._dummy_batch) batch_for_postproc.count = self._dummy_batch.count + self.exploration.postprocess_trajectory(self, batch_for_postproc) postprocessed_batch = self.postprocess_trajectory(batch_for_postproc) if state_outs: B = 4 # For RNNs, have B=4, T=[depends on sample_batch_size] @@ -700,27 +712,33 @@ class Policy(metaclass=ABCMeta): ret[view_col] = \ np.zeros((batch_size, ) + shape[1:], np.float32) else: - ret[view_col] = np.zeros_like( - [view_req.space.sample() for _ in range(batch_size)]) + if isinstance(view_req.space, gym.spaces.Space): + ret[view_col] = np.zeros_like( + [view_req.space.sample() for _ in range(batch_size)]) + else: + ret[view_col] = [view_req.space for _ in range(batch_size)] + return SampleBatch(ret) def _update_model_inference_view_requirements_from_init_state(self): - """Uses this Model's initial state to auto-add necessary ViewReqs. + """Uses Model's (or this Policy's) init state to add needed ViewReqs. Can be called from within a Policy to make sure RNNs automatically update their internal state-related view requirements. Changes the `self.inference_view_requirements` dict. """ - model = self.model + self._model_init_state_automatically_added = True + model = getattr(self, "model", None) + obj = model or self # Add state-ins to this model's view. - for i, state in enumerate(model.get_initial_state()): - model.inference_view_requirements["state_in_{}".format(i)] = \ - ViewRequirement( - "state_out_{}".format(i), - data_rel_pos=-1, - space=Box(-1.0, 1.0, shape=state.shape)) - model.inference_view_requirements["state_out_{}".format(i)] = \ - ViewRequirement(space=Box(-1.0, 1.0, shape=state.shape)) + for i, state in enumerate(obj.get_initial_state()): + space = Box(-1.0, 1.0, shape=state.shape) if \ + hasattr(state, "shape") else state + view_reqs = model.inference_view_requirements if model else \ + self.view_requirements + view_reqs["state_in_{}".format(i)] = ViewRequirement( + "state_out_{}".format(i), shift=-1, space=space) + view_reqs["state_out_{}".format(i)] = ViewRequirement(space=space) def clip_action(action, action_space): diff --git a/rllib/policy/sample_batch.py b/rllib/policy/sample_batch.py index db5ead008..a2934fdb9 100644 --- a/rllib/policy/sample_batch.py +++ b/rllib/policy/sample_batch.py @@ -115,7 +115,9 @@ class SampleBatch: [s[k] for s in concat_samples], time_major=concat_samples[0].time_major) return SampleBatch( - out, _seq_lens=seq_lens, _time_major=concat_samples[0].time_major) + out, + _seq_lens=np.array(seq_lens, dtype=np.int32), + _time_major=concat_samples[0].time_major) @PublicAPI def concat(self, other: "SampleBatch") -> "SampleBatch": @@ -154,7 +156,8 @@ class SampleBatch: """ return SampleBatch( {k: np.array(v, copy=True) - for (k, v) in self.data.items()}) + for (k, v) in self.data.items()}, + _seq_lens=self.seq_lens) @PublicAPI def rows(self) -> Dict[str, TensorType]: @@ -414,16 +417,17 @@ class MultiAgentBatch: Args: policy_batches (Dict[PolicyID, SampleBatch]): Mapping from policy ids to SampleBatches of experiences. - env_steps (int): The number of timesteps in the environment this - batch contains. This will be less than the number of + env_steps (int): The number of environment steps in the environment + this batch contains. This will be less than the number of transitions this batch contains across all policies in total. """ for v in policy_batches.values(): assert isinstance(v, SampleBatch) self.policy_batches = policy_batches - # Called count for uniformity with SampleBatch. Prefer to access this - # via the env_steps() method when possible for clarity. + # Called "count" for uniformity with SampleBatch. + # Prefer to access this via the `env_steps()` method when possible + # for clarity. self.count = env_steps @PublicAPI @@ -523,7 +527,8 @@ class MultiAgentBatch: """ if len(policy_batches) == 1 and DEFAULT_POLICY_ID in policy_batches: return policy_batches[DEFAULT_POLICY_ID] - return MultiAgentBatch(policy_batches, env_steps) + return MultiAgentBatch( + policy_batches=policy_batches, env_steps=env_steps) @staticmethod @PublicAPI diff --git a/rllib/policy/tests/test_compute_log_likelihoods.py b/rllib/policy/tests/test_compute_log_likelihoods.py index da83ae10d..b64eabd47 100644 --- a/rllib/policy/tests/test_compute_log_likelihoods.py +++ b/rllib/policy/tests/test_compute_log_likelihoods.py @@ -2,6 +2,7 @@ import numpy as np from scipy.stats import norm import unittest +import ray import ray.rllib.agents.dqn as dqn import ray.rllib.agents.pg as pg import ray.rllib.agents.ppo as ppo @@ -87,8 +88,8 @@ def do_test_log_likelihood(run, logp = policy.compute_log_likelihoods( np.array([a]), preprocessed_obs_batch, - prev_action_batch=np.array([prev_a]), - prev_reward_batch=np.array([prev_r])) + prev_action_batch=np.array([prev_a]) if prev_a else None, + prev_reward_batch=np.array([prev_r]) if prev_r else None) check(logp, expected_logp[0], rtol=0.2) # Test all available actions for their logp values. else: @@ -98,12 +99,20 @@ def do_test_log_likelihood(run, logp = policy.compute_log_likelihoods( np.array([a]), preprocessed_obs_batch, - prev_action_batch=np.array([prev_a]), - prev_reward_batch=np.array([prev_r])) + prev_action_batch=np.array([prev_a]) if prev_a else None, + prev_reward_batch=np.array([prev_r]) if prev_r else None) check(np.exp(logp), expected_prob, atol=0.2) class TestComputeLogLikelihood(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + ray.init() + + @classmethod + def tearDownClass(cls) -> None: + ray.shutdown() + def test_dqn(self): """Tests, whether DQN correctly computes logp in soft-q mode.""" config = dqn.DEFAULT_CONFIG.copy() diff --git a/rllib/policy/tf_policy.py b/rllib/policy/tf_policy.py index a5d6d6bf4..f6e48dad2 100644 --- a/rllib/policy/tf_policy.py +++ b/rllib/policy/tf_policy.py @@ -188,7 +188,8 @@ class TFPolicy(Policy): self._apply_op = None self._stats_fetches = {} self._timestep = timestep if timestep is not None else \ - tf1.placeholder(tf.int64, (), name="timestep") + tf1.placeholder_with_default( + tf.zeros((), dtype=tf.int64), (), name="timestep") self._optimizer = None self._grads_and_vars = None @@ -274,7 +275,8 @@ class TFPolicy(Policy): else: self._loss = loss - self._optimizer = self.optimizer() + if self._optimizer is None: + self._optimizer = self.optimizer() self._grads_and_vars = [ (g, v) for (g, v) in self.gradients(self._optimizer, self._loss) if g is not None diff --git a/rllib/policy/tf_policy_template.py b/rllib/policy/tf_policy_template.py index 2a738ddc9..a4f5e12b2 100644 --- a/rllib/policy/tf_policy_template.py +++ b/rllib/policy/tf_policy_template.py @@ -8,7 +8,6 @@ from ray.rllib.policy import eager_tf_policy from ray.rllib.policy.policy import Policy, LEARNER_STATS_KEY from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.policy.tf_policy import TFPolicy -from ray.rllib.policy.view_requirement import ViewRequirement from ray.rllib.utils import add_mixins, force_list from ray.rllib.utils.annotations import override, DeveloperAPI from ray.rllib.utils.framework import try_import_tf @@ -66,8 +65,6 @@ def build_tf_policy( Policy, ModelV2, TensorType, TensorType, TensorType ], Tuple[TensorType, type, List[TensorType]]]] = None, mixins: Optional[List[type]] = None, - view_requirements_fn: Optional[Callable[[Policy], Dict[ - str, ViewRequirement]]] = None, get_batch_divisibility_req: Optional[Callable[[Policy], int]] = None, # TODO: (sven) deprecate once _use_trajectory_view_api is always True. obs_include_prev_action_reward: bool = True, @@ -231,7 +228,6 @@ def build_tf_policy( action_distribution_fn=action_distribution_fn, existing_inputs=existing_inputs, existing_model=existing_model, - view_requirements_fn=view_requirements_fn, get_batch_divisibility_req=get_batch_divisibility_req, obs_include_prev_action_reward=obs_include_prev_action_reward) diff --git a/rllib/policy/torch_policy_template.py b/rllib/policy/torch_policy_template.py index dd2aadfa6..2c698fefa 100644 --- a/rllib/policy/torch_policy_template.py +++ b/rllib/policy/torch_policy_template.py @@ -8,7 +8,6 @@ from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 from ray.rllib.policy.policy import Policy, LEARNER_STATS_KEY from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.policy.torch_policy import TorchPolicy -from ray.rllib.policy.view_requirement import ViewRequirement from ray.rllib.utils import add_mixins, force_list from ray.rllib.utils.annotations import override, DeveloperAPI from ray.rllib.utils.framework import try_import_torch @@ -70,8 +69,6 @@ def build_torch_policy( apply_gradients_fn: Optional[Callable[ [Policy, "torch.optim.Optimizer"], None]] = None, mixins: Optional[List[type]] = None, - view_requirements_fn: Optional[Callable[[Policy], Dict[ - str, ViewRequirement]]] = None, get_batch_divisibility_req: Optional[Callable[[Policy], int]] = None ) -> Type[TorchPolicy]: """Helper function for creating a torch policy class at runtime. @@ -174,9 +171,6 @@ def build_torch_policy( mixins (Optional[List[type]]): Optional list of any class mixins for the returned policy class. These mixins will be applied in order and will have higher precedence than the TorchPolicy class. - view_requirements_fn (Optional[Callable[[Policy], - Dict[str, ViewRequirement]]]): An optional callable to retrieve - additional train view requirements for this policy. get_batch_divisibility_req (Optional[Callable[[Policy], int]]): Optional callable that returns the divisibility requirement for sample batches. If None, will assume a value of 1. @@ -242,9 +236,6 @@ def build_torch_policy( get_batch_divisibility_req=get_batch_divisibility_req, ) - # Update this Policy's ViewRequirements (if function given). - if callable(view_requirements_fn): - self.view_requirements.update(view_requirements_fn(self)) # Merge Model's view requirements into Policy's. self.view_requirements.update( self.model.inference_view_requirements) diff --git a/rllib/policy/view_requirement.py b/rllib/policy/view_requirement.py index 8813c3aca..f9c7750d4 100644 --- a/rllib/policy/view_requirement.py +++ b/rllib/policy/view_requirement.py @@ -29,31 +29,42 @@ class ViewRequirement: def __init__(self, data_col: Optional[str] = None, space: gym.Space = None, - data_rel_pos: Union[int, List[int]] = 0, + shift: Union[int, List[int]] = 0, + index: Optional[int] = None, used_for_training: bool = True): """Initializes a ViewRequirement object. Args: - data_col (): The data column name from the SampleBatch (str key). - If None, use the dict key under which this ViewRequirement - resides. + data_col (Optional[str]): The data column name from the SampleBatch + (str key). If None, use the dict key under which this + ViewRequirement resides. space (gym.Space): The gym Space used in case we need to pad data in inaccessible areas of the trajectory (t<0 or t>H). Default: Simple box space, e.g. rewards. - data_rel_pos (Union[int, str, List[int]]): Single shift value or + shift (Union[int, str, List[int]]): Single shift value or list of relative positions to use (relative to the underlying `data_col`). Example: For a view column "prev_actions", you can set - `data_col="actions"` and `data_rel_pos=-1`. + `data_col="actions"` and `shift=-1`. Example: For a view column "obs" in an Atari framestacking fashion, you can set `data_col="obs"` and - `data_rel_pos=[-3, -2, -1, 0]`. + `shift=[-3, -2, -1, 0]`. + Example: For the obs input to an attention net, you can specify + a range via a str: `shift="-100:0"`, which will pass in + the past 100 observations plus the current one. + index (Optional[int]): An optional absolute position arg, + used e.g. for the location of a requested inference dict within + the trajectory. Negative values refer to counting from the end + of a trajectory. used_for_training (bool): Whether the data will be used for training. If False, the column will not be copied into the final train batch. """ self.data_col = data_col - self.space = space or gym.spaces.Box( + self.space = space if space is not None else gym.spaces.Box( float("-inf"), float("inf"), shape=()) - self.data_rel_pos = data_rel_pos + + self.index = index + + self.shift = shift self.used_for_training = used_for_training diff --git a/rllib/tests/test_execution.py b/rllib/tests/test_execution.py index 869101809..2c4be172a 100644 --- a/rllib/tests/test_execution.py +++ b/rllib/tests/test_execution.py @@ -151,10 +151,10 @@ def test_concat_batches(ray_start_regular_shared): def test_standardize(ray_start_regular_shared): workers = make_workers(0) a = ParallelRollouts(workers, mode="async") - b = a.for_each(StandardizeFields(["t"])) + b = a.for_each(StandardizeFields([SampleBatch.EPS_ID])) batch = next(b) - assert abs(np.mean(batch["t"])) < 0.001, batch - assert abs(np.std(batch["t"]) - 1.0) < 0.001, batch + assert abs(np.mean(batch[SampleBatch.EPS_ID])) < 0.001, batch + assert abs(np.std(batch[SampleBatch.EPS_ID]) - 1.0) < 0.001, batch def test_async_grads(ray_start_regular_shared): diff --git a/rllib/tests/test_external_env.py b/rllib/tests/test_external_env.py index 681d719ac..d35e5003d 100644 --- a/rllib/tests/test_external_env.py +++ b/rllib/tests/test_external_env.py @@ -9,8 +9,9 @@ from ray.rllib.agents.dqn import DQNTrainer from ray.rllib.agents.pg import PGTrainer from ray.rllib.evaluation.rollout_worker import RolloutWorker from ray.rllib.env.external_env import ExternalEnv -from ray.rllib.tests.test_rollout_worker import (BadPolicy, MockPolicy, - MockEnv) +from ray.rllib.evaluation.tests.test_rollout_worker import (BadPolicy, + MockPolicy) +from ray.rllib.examples.env.mock_env import MockEnv from ray.rllib.utils.test_utils import framework_iterator from ray.tune.registry import register_env diff --git a/rllib/tests/test_external_multi_agent_env.py b/rllib/tests/test_external_multi_agent_env.py index fe34f13fe..1f1e56cc0 100644 --- a/rllib/tests/test_external_multi_agent_env.py +++ b/rllib/tests/test_external_multi_agent_env.py @@ -5,8 +5,8 @@ import unittest import ray from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv from ray.rllib.evaluation.rollout_worker import RolloutWorker +from ray.rllib.evaluation.tests.test_rollout_worker import MockPolicy from ray.rllib.examples.env.multi_agent import BasicMultiAgent -from ray.rllib.tests.test_rollout_worker import MockPolicy from ray.rllib.tests.test_external_env import make_simple_serving SimpleMultiServing = make_simple_serving(True, ExternalMultiAgentEnv) diff --git a/rllib/tests/test_multi_agent_env.py b/rllib/tests/test_multi_agent_env.py index a8c5bf3be..a2c3dddff 100644 --- a/rllib/tests/test_multi_agent_env.py +++ b/rllib/tests/test_multi_agent_env.py @@ -7,11 +7,13 @@ import ray from ray.tune.registry import register_env from ray.rllib.agents.dqn.dqn_tf_policy import DQNTFPolicy from ray.rllib.agents.pg import PGTrainer +from ray.rllib.evaluation.episode import MultiAgentEpisode +from ray.rllib.evaluation.rollout_worker import get_global_worker from ray.rllib.examples.policy.random_policy import RandomPolicy from ray.rllib.examples.env.multi_agent import MultiAgentCartPole, \ BasicMultiAgent, EarlyDoneMultiAgent, RoundRobinMultiAgent -from ray.rllib.tests.test_rollout_worker import MockPolicy from ray.rllib.evaluation.rollout_worker import RolloutWorker +from ray.rllib.evaluation.tests.test_rollout_worker import MockPolicy from ray.rllib.env.base_env import _MultiAgentEnvToBaseEnv from ray.rllib.utils.numpy import one_hot from ray.rllib.utils.test_utils import check @@ -321,21 +323,31 @@ class TestMultiAgentEnv(unittest.TestCase): if episodes is not None: # Pretend we did a model-based rollout and want to return # the extra trajectory. - builder = episodes[0].new_batch_builder() - rollout_id = random.randint(0, 10000) - for t in range(5): - builder.add_values( - agent_id="extra_0", - policy_id="p1", # use p1 so we can easily check it - t=t, - eps_id=rollout_id, # new id for each rollout - obs=obs_batch[0], - actions=0, - rewards=0, - dones=t == 4, - infos={}, - new_obs=obs_batch[0]) - batch = builder.build_and_reset(episode=None) + env_id = episodes[0].env_id + fake_eps = MultiAgentEpisode( + episodes[0]._policies, episodes[0]._policy_mapping_fn, + lambda: None, lambda x: None, env_id) + builder = get_global_worker().sampler.sample_collector + agent_id = "extra_0" + policy_id = "p1" # use p1 so we can easily check it + builder.add_init_obs(fake_eps, agent_id, env_id, policy_id, + -1, obs_batch[0]) + for t in range(4): + builder.add_action_reward_next_obs( + episode_id=fake_eps.episode_id, + agent_id=agent_id, + env_id=env_id, + policy_id=policy_id, + agent_done=t == 3, + values=dict( + t=t, + actions=0, + rewards=0, + dones=t == 3, + infos={}, + new_obs=obs_batch[0])) + batch = builder.postprocess_episode( + episode=fake_eps, build=True) episodes[0].add_extra_batch(batch) # Just return zeros for actions @@ -350,12 +362,17 @@ class TestMultiAgentEnv(unittest.TestCase): "p0": (ModelBasedPolicy, obs_space, act_space, {}), "p1": (ModelBasedPolicy, obs_space, act_space, {}), }, + policy_config={"_use_trajectory_view_api": True}, policy_mapping_fn=lambda agent_id: "p0", rollout_fragment_length=5) batch = ev.sample() + # 5 environment steps (rollout_fragment_length). self.assertEqual(batch.count, 5) + # 10 agent steps for p0: 2 agents, both using p0 as their policy. self.assertEqual(batch.policy_batches["p0"].count, 10) - self.assertEqual(batch.policy_batches["p1"].count, 25) + # 20 agent steps for p1: Each time both(!) agents takes 1 step, + # p1 takes 4: 5 (rollout-fragment length) * 4 = 20 + self.assertEqual(batch.policy_batches["p1"].count, 20) def test_train_multi_agent_cartpole_single_policy(self): n = 10 diff --git a/rllib/tests/test_perf.py b/rllib/tests/test_perf.py index 90148b043..4a4d7bdad 100644 --- a/rllib/tests/test_perf.py +++ b/rllib/tests/test_perf.py @@ -4,7 +4,7 @@ import unittest import ray from ray.rllib.evaluation.rollout_worker import RolloutWorker -from ray.rllib.tests.test_rollout_worker import MockPolicy +from ray.rllib.evaluation.tests.test_rollout_worker import MockPolicy class TestPerf(unittest.TestCase): diff --git a/rllib/train.py b/rllib/train.py index a89a23f18..228dcbfbc 100755 --- a/rllib/train.py +++ b/rllib/train.py @@ -8,12 +8,19 @@ import yaml import ray from ray.cluster_utils import Cluster from ray.tune.config_parser import make_parser +from ray.tune.progress_reporter import CLIReporter, JupyterNotebookReporter from ray.tune.result import DEFAULT_RESULTS_DIR from ray.tune.resources import resources_to_json from ray.tune.tune import run_experiments from ray.tune.schedulers import create_scheduler from ray.rllib.utils.framework import try_import_tf, try_import_torch +try: + class_name = get_ipython().__class__.__name__ + IS_NOTEBOOK = True if "Terminal" not in class_name else False +except NameError: + IS_NOTEBOOK = False + # Try to import both backends for flag checking/warnings. tf1, tf, tfv = try_import_tf() torch, _ = try_import_torch() @@ -184,10 +191,10 @@ def run(args, parser): if args.v: exp["config"]["log_level"] = "INFO" - verbose = 2 + verbose = 3 # Print details on trial result if args.vv: exp["config"]["log_level"] = "DEBUG" - verbose = 3 + verbose = 3 # Print details on trial result if args.ray_num_nodes: cluster = Cluster() @@ -206,12 +213,19 @@ def run(args, parser): num_gpus=args.ray_num_gpus, local_mode=args.local_mode) + if IS_NOTEBOOK: + progress_reporter = JupyterNotebookReporter( + overwrite=verbose >= 3, print_intermediate_tables=verbose >= 1) + else: + progress_reporter = CLIReporter(print_intermediate_tables=verbose >= 1) + run_experiments( experiments, scheduler=create_scheduler(args.scheduler, **args.scheduler_config), resume=args.resume, queue_trials=args.queue_trials, verbose=verbose, + progress_reporter=progress_reporter, concurrent=True) ray.shutdown() diff --git a/rllib/tuned_examples/dqn/cartpole-dqn-softq.yaml b/rllib/tuned_examples/dqn/cartpole-dqn-softq.yaml new file mode 100644 index 000000000..c6919140b --- /dev/null +++ b/rllib/tuned_examples/dqn/cartpole-dqn-softq.yaml @@ -0,0 +1,16 @@ +cartpole-dqn: + env: CartPole-v0 + run: DQN + stop: + episode_reward_mean: 150 + timesteps_total: 100000 + config: + # Works for both torch and tf. + framework: tf + model: + fcnet_hiddens: [64] + fcnet_activation: linear + n_step: 3 + exploration_config: + type: SoftQ + temperature: 0.5 \ No newline at end of file diff --git a/rllib/utils/exploration/tests/test_parameter_noise.py b/rllib/utils/exploration/tests/test_parameter_noise.py index c0996a095..2925c050f 100644 --- a/rllib/utils/exploration/tests/test_parameter_noise.py +++ b/rllib/utils/exploration/tests/test_parameter_noise.py @@ -1,12 +1,21 @@ import numpy as np import unittest +import ray import ray.rllib.agents.ddpg as ddpg import ray.rllib.agents.dqn as dqn from ray.rllib.utils.test_utils import check, framework_iterator class TestParameterNoise(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + ray.init() + + @classmethod + def tearDownClass(cls) -> None: + ray.shutdown() + def test_ddpg_parameter_noise(self): self.do_test_parameter_noise_exploration( ddpg.DDPGTrainer, ddpg.DEFAULT_CONFIG, "Pendulum-v0", {}, @@ -37,6 +46,10 @@ class TestParameterNoise(unittest.TestCase): trainer = trainer_cls(config=config, env=env) policy = trainer.get_policy() pol_sess = getattr(policy, "_sess", None) + # Remove noise that has been added during policy initialization + # (exploration.postprocess_trajectory does add noise to measure + # the delta). + policy.exploration._remove_noise(tf_sess=pol_sess) self.assertFalse(policy.exploration.weights_are_currently_noisy) noise_before = self._get_current_noise(policy, fw) @@ -96,6 +109,12 @@ class TestParameterNoise(unittest.TestCase): config["explore"] = False trainer = trainer_cls(config=config, env=env) policy = trainer.get_policy() + pol_sess = getattr(policy, "_sess", None) + # Remove noise that has been added during policy initialization + # (exploration.postprocess_trajectory does add noise to measure + # the delta). + policy.exploration._remove_noise(tf_sess=pol_sess) + self.assertFalse(policy.exploration.weights_are_currently_noisy) initial_weights = self._get_current_weight(policy, fw) diff --git a/rllib/utils/framework.py b/rllib/utils/framework.py index 9025b8736..1d0a8afcd 100644 --- a/rllib/utils/framework.py +++ b/rllib/utils/framework.py @@ -88,6 +88,7 @@ def try_import_tf(error=False): tf1_module = tf_module.compat.v1 if not was_imported: tf1_module.disable_v2_behavior() + tf1_module.enable_resource_variables() # No compat.v1 -> return tf as is. except AttributeError: tf1_module = tf_module @@ -281,7 +282,7 @@ def get_activation_fn(name: Optional[str] = None, framework: str = "tf"): elif framework == "jax": if name in ["linear", None]: return None - jax = try_import_jax() + jax, flax = try_import_jax() if name == "swish": return jax.nn.swish if name == "relu": diff --git a/rllib/utils/test_utils.py b/rllib/utils/test_utils.py index 05713c7cb..5460d9c27 100644 --- a/rllib/utils/test_utils.py +++ b/rllib/utils/test_utils.py @@ -5,7 +5,7 @@ import numpy as np from ray.rllib.utils.framework import try_import_jax, try_import_tf, \ try_import_torch -jax = try_import_jax() +jax, flax = try_import_jax() tf1, tf, tfv = try_import_tf() if tf1: eager_mode = None diff --git a/rllib/utils/typing.py b/rllib/utils/typing.py index 6684ae53a..592f0424d 100644 --- a/rllib/utils/typing.py +++ b/rllib/utils/typing.py @@ -67,6 +67,10 @@ EnvInfoDict = dict # Represents a File object FileType = Any +# Represents a ViewRequirements dict mapping column names (str) to +# ViewRequirement objects. +ViewRequirementsDict = Dict[str, "ViewRequirement"] + # Represents the result dict returned by Trainer.train(). ResultDict = dict diff --git a/src/ray/common/bundle_spec.cc b/src/ray/common/bundle_spec.cc index d9dd610ec..701fa30c2 100644 --- a/src/ray/common/bundle_spec.cc +++ b/src/ray/common/bundle_spec.cc @@ -25,6 +25,27 @@ void BundleSpecification::ComputeResources() { } else { unit_resource_.reset(new ResourceSet(unit_resource)); } + + // Generate placement group bundle labels. + ComputeBundleResourceLabels(); +} + +void BundleSpecification::ComputeBundleResourceLabels() { + RAY_CHECK(unit_resource_); + + for (const auto &resource_pair : unit_resource_->GetResourceMap()) { + double resource_value = resource_pair.second; + + /// With bundle index (e.g., CPU_group_i_zzz). + const std::string &resource_label = + FormatPlacementGroupResource(resource_pair.first, PlacementGroupId(), Index()); + bundle_resource_labels_.insert(std::make_pair(resource_label, resource_value)); + + /// Without bundle index (e.g., CPU_group_zzz). + const std::string &wildcard_label = + FormatPlacementGroupResource(resource_pair.first, PlacementGroupId(), -1); + bundle_resource_labels_.insert(std::make_pair(wildcard_label, resource_value)); + } } const ResourceSet &BundleSpecification::GetRequiredResources() const { diff --git a/src/ray/common/bundle_spec.h b/src/ray/common/bundle_spec.h index cbdfa3049..843770450 100644 --- a/src/ray/common/bundle_spec.h +++ b/src/ray/common/bundle_spec.h @@ -72,6 +72,11 @@ class BundleSpecification : public MessageWrapper { on_spillback_ = callback; } + /// Get all placement group bundle resource labels. + const std::unordered_map &GetFormattedResources() const { + return bundle_resource_labels_; + } + /// Returns the schedule bundle callback, or nullptr. const ScheduleBundleCallback &OnSchedule() const { return on_schedule_; } @@ -82,18 +87,27 @@ class BundleSpecification : public MessageWrapper { private: void ComputeResources(); + void ComputeBundleResourceLabels(); /// Field storing unit resources. Initialized in constructor. /// TODO(ekl) consider optimizing the representation of ResourceSet for fast copies /// instead of keeping shared pointers here. std::shared_ptr unit_resource_; + /// When a bundle is assigned on a node, we'll add the following special resources on + /// that node: + /// 1) `CPU_group_${group_id}`: this is the requested resource when the actor + /// or task specifies placement group without bundle id. + /// 2) `CPU_group_${bundle_index}_${group_id}`: this is the requested resource + /// when the actor or task specifies placement group with bundle id. + std::unordered_map bundle_resource_labels_; + mutable ScheduleBundleCallback on_schedule_ = nullptr; mutable SpillbackBundleCallback on_spillback_ = nullptr; }; -/// Format a placement group resource, e.g., CPU -> CPU_group_YYY_i +/// Format a placement group resource, e.g., CPU -> CPU_group_i std::string FormatPlacementGroupResource(const std::string &original_resource_name, const PlacementGroupID &group_id, int64_t bundle_index = -1); diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 11770be3a..a1af82469 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -32,14 +32,10 @@ RAY_CONFIG(int64_t, ray_cookie, 0x5241590000000000) /// The duration that a single handler on the event loop can take before a /// warning is logged that the handler is taking too long. -RAY_CONFIG(int64_t, handler_warning_timeout_ms, 100) +RAY_CONFIG(int64_t, handler_warning_timeout_ms, 1000) /// The duration between heartbeats sent by the raylets. RAY_CONFIG(int64_t, raylet_heartbeat_timeout_milliseconds, 100) -/// Whether to send heartbeat lightly. When it is enalbed, only changed part, -/// like should_global_gc or changed resources, will be included in the heartbeat, -/// and gcs only broadcast the changed heartbeat. -RAY_CONFIG(bool, light_heartbeat_enabled, true) /// If a component has not sent a heartbeat in the last num_heartbeats_timeout /// heartbeat intervals, the raylet monitor process will report /// it as dead to the db_client table. @@ -49,6 +45,12 @@ RAY_CONFIG(int64_t, num_heartbeats_timeout, 300) /// handler is drifting. RAY_CONFIG(uint64_t, num_heartbeats_warning, 5) +/// The duration between reporting resources sent by the raylets. +RAY_CONFIG(int64_t, raylet_report_resources_period_milliseconds, 100) +/// Whether to report resource usage lightly. When it is enalbed, only changed part, +/// like should_global_gc or changed resources, will be included in the message. +RAY_CONFIG(bool, light_report_resource_usage_enabled, true) + /// The duration between dumping debug info to logs, or -1 to disable. RAY_CONFIG(int64_t, debug_dump_period_milliseconds, 10000) @@ -175,6 +177,10 @@ RAY_CONFIG(int64_t, worker_register_timeout_seconds, 30) RAY_CONFIG(int64_t, redis_db_connect_retries, 50) RAY_CONFIG(int64_t, redis_db_connect_wait_milliseconds, 100) +/// Timeout, in milliseconds, to wait before retrying a failed pull in the +/// ObjectManager. +RAY_CONFIG(int, object_manager_timer_freq_ms, 100) + /// Timeout, in milliseconds, to wait before retrying a failed pull in the /// ObjectManager. RAY_CONFIG(int, object_manager_pull_timeout_ms, 10000) @@ -196,15 +202,6 @@ RAY_CONFIG(uint64_t, object_manager_default_chunk_size, 5 * 1024 * 1024) /// excessive memory usage during object broadcast to many receivers. RAY_CONFIG(uint64_t, object_manager_max_bytes_in_flight, 2L * 1024 * 1024 * 1024) -/// Number of workers per Python worker process -RAY_CONFIG(int, num_workers_per_process_python, 1) - -/// Number of workers per Java worker process -RAY_CONFIG(int, num_workers_per_process_java, 1) - -/// Number of workers per CPP worker process -RAY_CONFIG(int, num_workers_per_process_cpp, 1) - /// Maximum number of ids in one batch to send to GCS to delete keys. RAY_CONFIG(uint32_t, maximum_gcs_deletion_batch_size, 1000) @@ -251,6 +248,12 @@ RAY_CONFIG(int32_t, object_store_full_max_retries, 5) /// This will be exponentially increased for each retry. RAY_CONFIG(uint32_t, object_store_full_initial_delay_ms, 1000) +/// The amount of time to wait between logging plasma space usage debug messages. +RAY_CONFIG(uint64_t, object_store_usage_log_interval_s, 10 * 60) + +/// The amount of time between automatic local Python GC triggers. +RAY_CONFIG(uint64_t, local_gc_interval_s, 10 * 60) + /// Duration to wait between retries for failed tasks. RAY_CONFIG(uint32_t, task_retry_delay_ms, 5000) @@ -301,11 +304,6 @@ RAY_CONFIG(bool, report_worker_backlog, true) /// The timeout for synchronous GCS requests in seconds. RAY_CONFIG(int64_t, gcs_server_request_timeout_seconds, 5) -/// Whether to enable multi tenancy features. -RAY_CONFIG(bool, enable_multi_tenancy, - getenv("RAY_ENABLE_MULTI_TENANCY") == nullptr || - getenv("RAY_ENABLE_MULTI_TENANCY") == std::string("1")) - /// Whether to enable worker prestarting: https://github.com/ray-project/ray/issues/12052 RAY_CONFIG(bool, enable_worker_prestart, getenv("RAY_ENABLE_WORKER_PRESTART") == nullptr || diff --git a/src/ray/common/task/scheduling_resources.cc b/src/ray/common/task/scheduling_resources.cc index e5080cab5..db7be28b6 100644 --- a/src/ray/common/task/scheduling_resources.cc +++ b/src/ray/common/task/scheduling_resources.cc @@ -4,7 +4,6 @@ #include #include "absl/container/flat_hash_map.h" -#include "ray/common/bundle_spec.h" #include "ray/util/logging.h" namespace ray { @@ -228,51 +227,6 @@ void ResourceSet::AddResources(const ResourceSet &other) { } } -void ResourceSet::CommitBundleResources(const PlacementGroupID &group_id, - const int bundle_index, - const ResourceSet &other) { - for (const auto &resource_pair : other.GetResourceAmountMap()) { - // With bundle index (e.g., CPU_group_i_zzz). - const std::string &resource_label = - FormatPlacementGroupResource(resource_pair.first, group_id, bundle_index); - const FractionalResourceQuantity &resource_capacity = resource_pair.second; - resource_capacity_[resource_label] += resource_capacity; - - // Without bundle index (e.g., CPU_group_zzz). - const std::string &wildcard_label = - FormatPlacementGroupResource(resource_pair.first, group_id, -1); - resource_capacity_[wildcard_label] += resource_capacity; - } -} - -void ResourceSet::ReturnBundleResources(const PlacementGroupID &group_id, - const int bundle_index) { - absl::flat_hash_map to_restore; - for (auto iter = resource_capacity_.begin(); iter != resource_capacity_.end();) { - const std::string &bundle_resource_label = iter->first; - // We only consider the indexed resources, ignoring the wildcard resource. - // This is because when multiple bundles are created on one node, the quantity - // of the wildcard resources contains resources from multiple bundles. - if (IsBundleIndex(bundle_resource_label, group_id, bundle_index)) { - const std::string &resource_label = GetOriginalResourceName(bundle_resource_label); - const FractionalResourceQuantity &resource_capacity = iter->second; - to_restore[resource_label] = resource_capacity; - iter = resource_capacity_.erase(iter); - } else { - iter++; - } - } - // For each matching resource to restore (e.g., key like CPU, GPU). - for (const auto &pair : to_restore) { - resource_capacity_[pair.first] += pair.second; - auto wildcard_resource = FormatPlacementGroupResource(pair.first, group_id, -1); - resource_capacity_[wildcard_resource] -= pair.second; - if (resource_capacity_[wildcard_resource] <= 0) { - resource_capacity_.erase(wildcard_resource); - } - } -} - FractionalResourceQuantity ResourceSet::GetResource( const std::string &resource_name) const { if (resource_capacity_.count(resource_name) == 0) { @@ -686,43 +640,10 @@ void ResourceIdSet::AddOrUpdateResource(const std::string &resource_name, } } -void ResourceIdSet::CommitBundleResourceIds(const PlacementGroupID &group_id, - const int bundle_index, - const std::string &resource_name, - ResourceIds &resource_ids) { - auto index_name = FormatPlacementGroupResource(resource_name, group_id, bundle_index); - auto wildcard_name = FormatPlacementGroupResource(resource_name, group_id, -1); - available_resources_[index_name] = available_resources_[index_name].Plus(resource_ids); - available_resources_[wildcard_name] = - available_resources_[wildcard_name].Plus(resource_ids); -} - -void ResourceIdSet::ReturnBundleResources(const PlacementGroupID &group_id, - const int bundle_index, - const std::string &original_resource_name) { - auto index_resource_name = - FormatPlacementGroupResource(original_resource_name, group_id, bundle_index); - auto iter_index = available_resources_.find(index_resource_name); - if (iter_index == available_resources_.end()) { - return; - } - - // Erase and transfer the index bundle resource back to the original. - auto bundle_ids = iter_index->second; - available_resources_.erase(iter_index); - available_resources_[original_resource_name] = - (available_resources_[original_resource_name].Plus(bundle_ids)); - - // Also erase the the equivalent number of units from the wildcard resource. - auto wildcard_name = FormatPlacementGroupResource(original_resource_name, group_id, -1); - available_resources_[wildcard_name].Acquire(bundle_ids.TotalQuantity()); - if (available_resources_[wildcard_name].TotalQuantityIsZero()) { - available_resources_.erase(wildcard_name); - } -} - void ResourceIdSet::DeleteResource(const std::string &resource_name) { - available_resources_.erase(resource_name); + if (available_resources_.count(resource_name) != 0) { + available_resources_.erase(resource_name); + } } const std::unordered_map &ResourceIdSet::AvailableResources() @@ -848,6 +769,14 @@ void SchedulingResources::Acquire(const ResourceSet &resources) { resources_available_.SubtractResourcesStrict(resources); } +// The reason we need this method is sometimes we may want add some converted +// resource which is not exist in total resource to the available resource. +// (e.g., placement group) +void SchedulingResources::AddResource(const ResourceSet &resources) { + resources_total_.AddResources(resources); + resources_available_.AddResources(resources); +} + void SchedulingResources::UpdateResourceCapacity(const std::string &resource_name, int64_t capacity) { const FractionalResourceQuantity new_capacity = FractionalResourceQuantity(capacity); @@ -873,26 +802,6 @@ void SchedulingResources::UpdateResourceCapacity(const std::string &resource_nam } } -void SchedulingResources::PrepareBundleResources(const PlacementGroupID &group, - const int bundle_index, - const ResourceSet &resource_set) { - resources_available_.SubtractResourcesStrict(resource_set); - resources_total_.SubtractResourcesStrict(resource_set); -} - -void SchedulingResources::CommitBundleResources(const PlacementGroupID &group, - const int bundle_index, - const ResourceSet &resource_set) { - resources_available_.CommitBundleResources(group, bundle_index, resource_set); - resources_total_.CommitBundleResources(group, bundle_index, resource_set); -} - -void SchedulingResources::ReturnBundleResources(const PlacementGroupID &group_id, - const int bundle_index) { - resources_available_.ReturnBundleResources(group_id, bundle_index); - resources_total_.ReturnBundleResources(group_id, bundle_index); -} - void SchedulingResources::DeleteResource(const std::string &resource_name) { resources_total_.DeleteResource(resource_name); resources_available_.DeleteResource(resource_name); diff --git a/src/ray/common/task/scheduling_resources.h b/src/ray/common/task/scheduling_resources.h index 44fecfe40..41ed07a1c 100644 --- a/src/ray/common/task/scheduling_resources.h +++ b/src/ray/common/task/scheduling_resources.h @@ -148,33 +148,6 @@ class ResourceSet { /// \return Void. void AddResources(const ResourceSet &other); - /// \brief Aggregate resources from the other set into this set, adding any missing - /// resource labels to this set. - /// - /// This adds both the the indexed and wildcard resources (e.g., both - /// CPU_group_i_zzz and CPU_group_zzz). - /// - /// NOTE: This method should be used AFTER resources are COMMITTED. - /// It can have unexpected behavior if you call this method on PREPARED resources. - /// - /// \param group_id: The placement group id. - /// \param bundle_index: The index of the bundle. - /// \param other: The other resource set to add. - /// \return Void. - void CommitBundleResources(const PlacementGroupID &group_id, const int bundle_index, - const ResourceSet &other); - - /// \brief Return back all the bundle resource. Changing the resource name and adding - /// any missing resource labels to this set. - /// - /// Note that this method assumes bundle resources are COMMITTED. - /// Please make sure to commit bundle resources before calling this method. - /// - /// \param group_id: The placement group id. - /// \param bundle_index: The bundle index to return resources for. - /// \return Void. - void ReturnBundleResources(const PlacementGroupID &group_id, const int bundle_index); - /// \brief Subtract a set of resources from the current set of resources and /// check that the post-subtraction result nonnegative. Assumes other /// is a subset of the ResourceSet. Deletes any resource if the capacity after @@ -431,31 +404,6 @@ class ResourceIdSet { /// \param capacity capacity of the resource being added void AddOrUpdateResource(const std::string &resource_name, int64_t capacity); - /// \brief Commit a Bundle resource in the ResourceIdSet. - /// - /// This adds both the the indexed and wildcard resources (e.g., both - /// CPU_group_i_zzz and CPU_group_zzz). - /// - /// \param group_id: The placement group id. - /// \param bundle_index: The index of the bundle. - /// \param resource_name the name of the resource to create/update (e.g., "CPU"). - /// \param resource_ids resource_ids of the resource being added - void CommitBundleResourceIds(const PlacementGroupID &group_id, const int bundle_index, - const std::string &resource_name, - ResourceIds &resource_ids); - - /// \brief remove a Bundle resource in the ResourceIdSet. - /// - /// The bundle resources will be returned to their original resource names. - /// Note that the bundle resources should've been COMMITTED before this method is - /// called. - /// - /// \param group_id: The placement group id. - /// \param bundle_index: The index of the bundle. - /// \param resource_name the name of the resource to remove (e.g., "CPU"). - void ReturnBundleResources(const PlacementGroupID &group_id, const int bundle_index, - const std::string &resource_name); - /// \brief Deletes a resource in the ResourceIdSet. This does not raise an exception, /// just deletes the resource. Tasks with acquired resources keep running. /// @@ -564,6 +512,12 @@ class SchedulingResources { /// \return Void. void Acquire(const ResourceSet &resources); + /// \brief Add a new resource to available resource. + /// + /// \param resources: the amount of resources to be added. + /// \return Void. + void AddResource(const ResourceSet &resources); + /// Returns debug string for class. /// /// \return string. @@ -577,42 +531,6 @@ class SchedulingResources { /// \return Void. void UpdateResourceCapacity(const std::string &resource_name, int64_t capacity); - /// \brief Update total, available and load resources with the ResourceIds. - /// Create if not exists. This will only update resources, but it won't - /// create placement group resources. That'll be done when resources are - /// COMMITTED. Commit should be done by CommitBundleResources. - /// - /// We need this step for running 2PC protocol for atomic placement group creation. - /// - /// \param resource_name: Name of the resource to be modified. - /// \param resource_set: New resource_set of the resource. - void PrepareBundleResources(const PlacementGroupID &group, const int bundle_index, - const ResourceSet &resource_set); - - /// \brief Commit placement group resources. It means this method'll create - /// placement group resources. The original resources should've been updated - /// by PrepareBundleResources. - /// - /// We need this step for running 2PC protocol for atomic placement group creation. - /// - /// The resources will be transfered from their original resource names. - /// This includes both the the indexed and wildcard resources (e.g., both - /// CPU_group_i_zzz and CPU_group_zzz). - /// - /// \param resource_name: Name of the resource to be modified - /// \param resource_set: New resource_set of the resource. - void CommitBundleResources(const PlacementGroupID &group, const int bundle_index, - const ResourceSet &resource_set); - - /// \brief delete total, available and load resources with the ResourceIds. - /// - /// The bundle resources will be returned to their original resource names. - /// This is the inverse of TransferToBundleResources(). - /// - /// \param group_id: Placement group id to delete resources for. - /// \param bundle_index: The bundle index to return resources for. - void ReturnBundleResources(const PlacementGroupID &group_id, const int bundle_index); - /// \brief Delete resource from total, available and load resources. /// /// \param resource_name: Name of the resource to be deleted. diff --git a/src/ray/common/task/scheduling_resources_test.cc b/src/ray/common/task/scheduling_resources_test.cc deleted file mode 100644 index 120f9f124..000000000 --- a/src/ray/common/task/scheduling_resources_test.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/common/task/scheduling_resources.h" - -#include - -#include "gtest/gtest.h" -#include "ray/common/id.h" - -namespace ray { -class SchedulingResourcesTest : public ::testing::Test { - public: - void SetUp() override { - resource_set = std::make_shared(); - resource_id_set = std::make_shared(); - } - - protected: - std::shared_ptr resource_set; - std::shared_ptr resource_id_set; -}; - -TEST_F(SchedulingResourcesTest, CommitBundleResources) { - PlacementGroupID group_id = PlacementGroupID::FromRandom(); - std::vector resource_labels = {"CPU"}; - std::vector resource_capacity = {1.0}; - ResourceSet resource(resource_labels, resource_capacity); - resource_set->CommitBundleResources(group_id, 1, resource); - resource_labels.pop_back(); - resource_labels.push_back("CPU_group_1_" + group_id.Hex()); - resource_labels.push_back("CPU_group_" + group_id.Hex()); - resource_capacity.push_back(1.0); - ResourceSet result_resource(resource_labels, resource_capacity); - ASSERT_EQ(1, resource_set->IsEqual(result_resource)); -} - -TEST_F(SchedulingResourcesTest, AddBundleResource) { - PlacementGroupID group_id = PlacementGroupID::FromRandom(); - std::string wild_name = "CPU_group_" + group_id.Hex(); - std::string index_name = "CPU_group_1_" + group_id.Hex(); - std::vector whole_ids = {1, 2, 3}; - ResourceIds resource_ids(whole_ids); - resource_id_set->CommitBundleResourceIds(group_id, 1, "CPU", resource_ids); - ASSERT_EQ(2, resource_id_set->AvailableResources().size()); - for (auto res : resource_id_set->AvailableResources()) { - ASSERT_TRUE(res.first == wild_name || res.first == index_name) << res.first; - } -} - -TEST_F(SchedulingResourcesTest, ReturnBundleResources) { - PlacementGroupID group_id = PlacementGroupID::FromRandom(); - std::vector resource_labels = {"CPU"}; - std::vector resource_capacity = {1.0}; - ResourceSet resource(resource_labels, resource_capacity); - resource_set->CommitBundleResources(group_id, 1, resource); - resource_labels.pop_back(); - resource_labels.push_back("CPU_group_" + group_id.Hex()); - resource_labels.push_back("CPU_group_1_" + group_id.Hex()); - resource_capacity.push_back(1.0); - ResourceSet result_resource(resource_labels, resource_capacity); - ASSERT_EQ(1, resource_set->IsEqual(result_resource)); - resource_set->ReturnBundleResources(group_id, 1); - ASSERT_EQ(1, resource_set->IsEqual(resource)) - << resource_set->ToString() << " vs " << resource.ToString(); -} - -TEST_F(SchedulingResourcesTest, MultipleBundlesAddRemove) { - PlacementGroupID group_id = PlacementGroupID::FromRandom(); - std::vector resource_labels = {"CPU"}; - std::vector resource_capacity = {1.0}; - ResourceSet resource(resource_labels, resource_capacity); - - // Construct resource set containing two bundles. - resource_set->CommitBundleResources(group_id, 1, resource); - resource_set->CommitBundleResources(group_id, 2, resource); - resource_labels = { - "CPU_group_" + group_id.Hex(), - "CPU_group_1_" + group_id.Hex(), - "CPU_group_2_" + group_id.Hex(), - }; - resource_capacity = {2.0, 1.0, 1.0}; - ResourceSet result_resource(resource_labels, resource_capacity); - ASSERT_EQ(1, resource_set->IsEqual(result_resource)) - << resource_set->ToString() << " vs " << result_resource.ToString(); - - // Return group 2. - resource_set->ReturnBundleResources(group_id, 2); - resource_labels = { - "CPU", - "CPU_group_" + group_id.Hex(), - "CPU_group_1_" + group_id.Hex(), - }; - resource_capacity = {1.0, 1.0, 1.0}; - ResourceSet result_resource2(resource_labels, resource_capacity); - ASSERT_EQ(1, resource_set->IsEqual(result_resource2)) - << resource_set->ToString() << " vs " << result_resource2.ToString(); - - // Return group 1. - resource_set->ReturnBundleResources(group_id, 1); - ASSERT_EQ(1, resource_set->IsEqual(ResourceSet({"CPU"}, {2.0}))) - << resource_set->ToString() << " vs " << resource.ToString(); -} - -TEST_F(SchedulingResourcesTest, MultipleBundlesAddRemoveIdSet) { - PlacementGroupID group_id = PlacementGroupID::FromRandom(); - ResourceIdSet resource_ids; - - // Construct resource set containing two bundles. - auto rid1 = ResourceIds({1, 2}); - auto rid2 = ResourceIds({3, 4}); - resource_ids.CommitBundleResourceIds(group_id, 1, "CPU", rid1); - resource_ids.CommitBundleResourceIds(group_id, 2, "CPU", rid2); - resource_ids.CommitBundleResourceIds(group_id, 1, "GPU", rid1); - resource_ids.CommitBundleResourceIds(group_id, 2, "GPU", rid2); - auto result = ResourceSet( - { - "CPU_group_" + group_id.Hex(), - "CPU_group_1_" + group_id.Hex(), - "CPU_group_2_" + group_id.Hex(), - "GPU_group_" + group_id.Hex(), - "GPU_group_1_" + group_id.Hex(), - "GPU_group_2_" + group_id.Hex(), - }, - {4.0, 2.0, 2.0, 4.0, 2.0, 2.0}); - ASSERT_EQ(1, resource_ids.ToResourceSet().IsEqual(result)) - << resource_ids.ToString() << " vs " << result.ToString(); - - // Remove the first bundle. - resource_ids.ReturnBundleResources(group_id, 1, "CPU"); - resource_ids.ReturnBundleResources(group_id, 1, "GPU"); - result = ResourceSet( - { - "CPU_group_" + group_id.Hex(), - "CPU", - "CPU_group_2_" + group_id.Hex(), - "GPU_group_" + group_id.Hex(), - "GPU", - "GPU_group_2_" + group_id.Hex(), - }, - {2.0, 2.0, 2.0, 2.0, 2.0, 2.0}); - ASSERT_EQ(1, resource_ids.ToResourceSet().IsEqual(result)) - << resource_ids.ToString() << " vs " << result.ToString(); - - // Remove the second bundle. - resource_ids.ReturnBundleResources(group_id, 2, "CPU"); - resource_ids.ReturnBundleResources(group_id, 2, "GPU"); - result = ResourceSet( - { - "CPU", - "GPU", - }, - {4.0, 4.0}); - ASSERT_EQ(1, resource_ids.ToResourceSet().IsEqual(result)) - << resource_ids.ToString() << " vs " << result.ToString(); -} - -} // namespace ray diff --git a/src/ray/common/task/task.cc b/src/ray/common/task/task.cc index c8ec17a17..bf0b8dfb1 100644 --- a/src/ray/common/task/task.cc +++ b/src/ray/common/task/task.cc @@ -35,6 +35,8 @@ void Task::CopyTaskExecutionSpec(const Task &task) { task_execution_spec_ = task.task_execution_spec_; } +void Task::SetBacklogSize(int64_t backlog_size) { backlog_size_ = backlog_size; } + int64_t Task::BacklogSize() const { return backlog_size_; } std::string Task::DebugString() const { diff --git a/src/ray/common/task/task.h b/src/ray/common/task/task.h index 1fe9e24fc..cc4010dcd 100644 --- a/src/ray/common/task/task.h +++ b/src/ray/common/task/task.h @@ -89,6 +89,8 @@ class Task { /// Returns the cancellation task callback, or nullptr. const CancelTaskCallback &OnCancellation() const { return on_cancellation_; } + void SetBacklogSize(int64_t backlog_size); + int64_t BacklogSize() const; std::string DebugString() const; diff --git a/src/ray/common/task/task_spec.cc b/src/ray/common/task/task_spec.cc index 875b61276..a07f5bade 100644 --- a/src/ray/common/task/task_spec.cc +++ b/src/ray/common/task/task_spec.cc @@ -193,6 +193,10 @@ const ResourceSet &TaskSpecification::GetRequiredPlacementResources() const { return *required_placement_resources_; } +std::string TaskSpecification::GetDebuggerBreakpoint() const { + return message_->debugger_breakpoint(); +} + std::unordered_map TaskSpecification::OverrideEnvironmentVariables() const { return MapFromProtobuf(message_->override_environment_variables()); diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index 2dec30283..bee66464c 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -131,6 +131,8 @@ class TaskSpecification : public MessageWrapper { /// \return The recomputed dependencies for the task. std::vector GetDependencies() const; + std::string GetDebuggerBreakpoint() const; + std::unordered_map OverrideEnvironmentVariables() const; bool IsDriverTask() const; diff --git a/src/ray/common/task/task_util.h b/src/ray/common/task/task_util.h index 825ae659e..a315762e1 100644 --- a/src/ray/common/task/task_util.h +++ b/src/ray/common/task/task_util.h @@ -87,6 +87,7 @@ class TaskSpecBuilder { const std::unordered_map &required_resources, const std::unordered_map &required_placement_resources, const BundleID &bundle_id, bool placement_group_capture_child_tasks, + const std::string &debugger_breakpoint, const std::unordered_map &override_environment_variables = {}) { message_->set_type(TaskType::NORMAL_TASK); @@ -108,6 +109,7 @@ class TaskSpecBuilder { message_->set_placement_group_bundle_index(bundle_id.second); message_->set_placement_group_capture_child_tasks( placement_group_capture_child_tasks); + message_->set_debugger_breakpoint(debugger_breakpoint); for (const auto &env : override_environment_variables) { (*message_->mutable_override_environment_variables())[env.first] = env.second; } @@ -146,7 +148,7 @@ class TaskSpecBuilder { /// /// \return Reference to the builder object itself. TaskSpecBuilder &SetActorCreationTaskSpec( - const ActorID &actor_id, int64_t max_restarts = 0, + const ActorID &actor_id, int64_t max_restarts = 0, int64_t max_task_retries = 0, const std::vector &dynamic_worker_options = {}, int max_concurrency = 1, bool is_detached = false, std::string name = "", bool is_asyncio = false, const std::string &extension_data = "") { @@ -154,6 +156,7 @@ class TaskSpecBuilder { auto actor_creation_spec = message_->mutable_actor_creation_task_spec(); actor_creation_spec->set_actor_id(actor_id.Binary()); actor_creation_spec->set_max_actor_restarts(max_restarts); + actor_creation_spec->set_max_task_retries(max_task_retries); for (const auto &option : dynamic_worker_options) { actor_creation_spec->add_dynamic_worker_options(option); } diff --git a/src/ray/common/test_util.cc b/src/ray/common/test_util.cc index bac0183d9..653955dde 100644 --- a/src/ray/common/test_util.cc +++ b/src/ray/common/test_util.cc @@ -148,8 +148,7 @@ std::string TestSetupUtil::StartRaylet(const std::string &store_socket_name, "--node_manager_port=" + std::to_string(port), "--node_ip_address=" + node_ip_address, "--redis_address=" + redis_address, "--redis_port=6379", "--min-worker-port=0", "--max-worker-port=0", - "--num_initial_workers=1", "--maximum_startup_concurrency=10", - "--static_resource_list=" + resource, + "--maximum_startup_concurrency=10", "--static_resource_list=" + resource, "--python_worker_command=" + CreateCommandLine({TEST_MOCK_WORKER_EXEC_PATH, store_socket_name, raylet_socket_name, std::to_string(port)}), diff --git a/src/ray/core_worker/actor_handle.cc b/src/ray/core_worker/actor_handle.cc index c59516080..73e56df54 100644 --- a/src/ray/core_worker/actor_handle.cc +++ b/src/ray/core_worker/actor_handle.cc @@ -58,6 +58,8 @@ ray::rpc::ActorHandle CreateInnerActorHandleFromActorTableData( inner.set_actor_cursor(task_spec.ReturnId(0).Binary()); inner.set_extension_data( actor_table_data.task_spec().actor_creation_task_spec().extension_data()); + inner.set_max_task_retries( + actor_table_data.task_spec().actor_creation_task_spec().max_task_retries()); return inner; } diff --git a/src/ray/core_worker/context.cc b/src/ray/core_worker/context.cc index b2a59ca41..685871c93 100644 --- a/src/ray/core_worker/context.cc +++ b/src/ray/core_worker/context.cc @@ -111,9 +111,7 @@ WorkerContext::WorkerContext(WorkerType worker_type, const WorkerID &worker_id, const JobID &job_id) : worker_type_(worker_type), worker_id_(worker_id), - current_job_id_(RayConfig::instance().enable_multi_tenancy() - ? job_id - : (worker_type_ == WorkerType::DRIVER ? job_id : JobID::Nil())), + current_job_id_(job_id), current_actor_id_(ActorID::Nil()), current_actor_placement_group_id_(PlacementGroupID::Nil()), placement_group_capture_child_tasks_(true), @@ -163,29 +161,17 @@ const std::unordered_map return override_environment_variables_; } -void WorkerContext::SetCurrentJobId(const JobID &job_id) { - RAY_CHECK(!RayConfig::instance().enable_multi_tenancy()); - current_job_id_ = job_id; -} - void WorkerContext::SetCurrentTaskId(const TaskID &task_id) { GetThreadContext().SetCurrentTaskId(task_id); } void WorkerContext::SetCurrentTask(const TaskSpecification &task_spec) { GetThreadContext().SetCurrentTask(task_spec); + RAY_CHECK(current_job_id_ == task_spec.JobId()); if (task_spec.IsNormalTask()) { - if (!RayConfig::instance().enable_multi_tenancy()) { - RAY_CHECK(current_job_id_.IsNil()); - SetCurrentJobId(task_spec.JobId()); - } current_task_is_direct_call_ = true; override_environment_variables_ = task_spec.OverrideEnvironmentVariables(); } else if (task_spec.IsActorCreationTask()) { - if (!RayConfig::instance().enable_multi_tenancy()) { - RAY_CHECK(current_job_id_.IsNil()); - SetCurrentJobId(task_spec.JobId()); - } RAY_CHECK(current_actor_id_.IsNil()); current_actor_id_ = task_spec.ActorCreationId(); current_actor_is_direct_call_ = true; @@ -196,21 +182,13 @@ void WorkerContext::SetCurrentTask(const TaskSpecification &task_spec) { placement_group_capture_child_tasks_ = task_spec.PlacementGroupCaptureChildTasks(); override_environment_variables_ = task_spec.OverrideEnvironmentVariables(); } else if (task_spec.IsActorTask()) { - if (!RayConfig::instance().enable_multi_tenancy()) { - RAY_CHECK(current_job_id_ == task_spec.JobId()); - } RAY_CHECK(current_actor_id_ == task_spec.ActorId()); } else { RAY_CHECK(false); } } -void WorkerContext::ResetCurrentTask(const TaskSpecification &task_spec) { - GetThreadContext().ResetCurrentTask(); - if (!RayConfig::instance().enable_multi_tenancy() && task_spec.IsNormalTask()) { - SetCurrentJobId(JobID::Nil()); - } -} +void WorkerContext::ResetCurrentTask() { GetThreadContext().ResetCurrentTask(); } std::shared_ptr WorkerContext::GetCurrentTask() const { return GetThreadContext().GetCurrentTask(); diff --git a/src/ray/core_worker/context.h b/src/ray/core_worker/context.h index 11c481b06..3b28cef3c 100644 --- a/src/ray/core_worker/context.h +++ b/src/ray/core_worker/context.h @@ -42,16 +42,12 @@ class WorkerContext { const std::unordered_map &GetCurrentOverrideEnvironmentVariables() const; - // TODO(kfstorm): Remove this once `enable_multi_tenancy` is deleted. - // TODO(edoakes): remove this once Python core worker uses the task interfaces. - void SetCurrentJobId(const JobID &job_id); - // TODO(edoakes): remove this once Python core worker uses the task interfaces. void SetCurrentTaskId(const TaskID &task_id); void SetCurrentTask(const TaskSpecification &task_spec); - void ResetCurrentTask(const TaskSpecification &task_spec); + void ResetCurrentTask(); std::shared_ptr GetCurrentTask() const; diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index fcff4356b..2aba250a5 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -39,14 +39,14 @@ void BuildCommonTaskSpec( const std::unordered_map &required_resources, const std::unordered_map &required_placement_resources, std::vector *return_ids, const ray::BundleID &bundle_id, - bool placement_group_capture_child_tasks, + bool placement_group_capture_child_tasks, const std::string debugger_breakpoint, const std::unordered_map &override_environment_variables) { // Build common task spec. builder.SetCommonTaskSpec( task_id, name, function.GetLanguage(), function.GetFunctionDescriptor(), job_id, current_task_id, task_index, caller_id, address, num_returns, required_resources, required_placement_resources, bundle_id, placement_group_capture_child_tasks, - override_environment_variables); + debugger_breakpoint, override_environment_variables); // Set task arguments. for (const auto &arg : args) { builder.AddArg(*arg); @@ -69,15 +69,7 @@ ray::JobID GetProcessJobID(const ray::CoreWorkerOptions &options) { if (options.worker_type == ray::WorkerType::WORKER) { // For workers, the job ID is assigned by Raylet via an environment variable. const char *job_id_env = std::getenv(kEnvVarKeyJobId); - // TODO(kfstorm): Use `RAY_CHECK` instead once the `enable_multi_tenancy` flag is - // removed. - // RAY_CHECK(job_id_env); - if (!job_id_env) { - // Multi-tenancy is disabled. - // NOTE(kfstorm): We can't read `RayConfig::instance().enable_multi_tenancy()` here - // because `RayConfig` is not initialized yet. - return ray::JobID::Nil(); - } + RAY_CHECK(job_id_env); return ray::JobID::FromHex(job_id_env); } return options.job_id; @@ -163,7 +155,7 @@ CoreWorkerProcess::CoreWorkerProcess(const CoreWorkerOptions &options) RAY_LOG(DEBUG) << "Stats setup in core worker."; // Initialize stats in core worker global tags. const ray::stats::TagsType global_tags = {{ray::stats::ComponentKey, "core_worker"}, - {ray::stats::VersionKey, "1.1.0.dev0"}}; + {ray::stats::VersionKey, "1.2.0.dev0"}}; // NOTE(lingxuan.zlx): We assume RayConfig is initialized before it's used. // RayConfig is generated in Java_io_ray_runtime_RayNativeRuntime_nativeInitialize @@ -429,7 +421,7 @@ CoreWorker::CoreWorker(const CoreWorkerOptions &options, const WorkerID &worker_ }; auto reconstruct_object_callback = [this](const ObjectID &object_id) { io_service_.post([this, object_id]() { - RAY_CHECK_OK(object_recovery_manager_->RecoverObject(object_id)); + RAY_CHECK(object_recovery_manager_->RecoverObject(object_id)); }); }; task_manager_.reset(new TaskManager( @@ -654,8 +646,11 @@ void CoreWorker::OnNodeRemoved(const rpc::GcsNodeInfo &node_info) { memory_store_->Delete(lost_objects); for (const auto &object_id : lost_objects) { RAY_LOG(INFO) << "Object " << object_id << " lost due to node failure " << node_id; - // The lost object must have been owned by us. - RAY_CHECK_OK(object_recovery_manager_->RecoverObject(object_id)); + // NOTE(swang): There is a race condition where this can return false if + // the reference went out of scope since the call to the ref counter to get + // the lost objects. It's okay to not mark the object as failed or recover + // the object since there are no reference holders. + static_cast(object_recovery_manager_->RecoverObject(object_id)); } } @@ -1187,8 +1182,10 @@ void CoreWorker::SpillOwnedObject(const ObjectID &object_id, // Find the raylet that hosts the primary copy of the object. NodeID pinned_at; bool spilled; - RAY_CHECK( - reference_counter_->IsPlasmaObjectPinnedOrSpilled(object_id, &pinned_at, &spilled)); + bool owned_by_us; + RAY_CHECK(reference_counter_->IsPlasmaObjectPinnedOrSpilled(object_id, &owned_by_us, + &pinned_at, &spilled)); + RAY_CHECK(owned_by_us); if (spilled) { // The object has already been spilled. return; @@ -1294,7 +1291,8 @@ void CoreWorker::SubmitTask(const RayFunction &function, const TaskOptions &task_options, std::vector *return_ids, int max_retries, BundleID placement_options, - bool placement_group_capture_child_tasks) { + bool placement_group_capture_child_tasks, + const std::string &debugger_breakpoint) { TaskSpecBuilder builder; const int next_task_index = worker_context_.GetNextTaskIndex(); const auto task_id = @@ -1320,7 +1318,7 @@ void CoreWorker::SubmitTask(const RayFunction &function, rpc_address_, function, args, task_options.num_returns, constrained_resources, required_resources, return_ids, placement_options, placement_group_capture_child_tasks, - override_environment_variables); + debugger_breakpoint, override_environment_variables); TaskSpecification task_spec = builder.Build(); if (options_.is_local_mode) { ExecuteTaskLocalMode(task_spec); @@ -1376,8 +1374,10 @@ Status CoreWorker::CreateActor(const RayFunction &function, new_placement_resources, &return_ids, actor_creation_options.placement_options, actor_creation_options.placement_group_capture_child_tasks, + "", /* debugger_breakpoint */ override_environment_variables); builder.SetActorCreationTaskSpec(actor_id, actor_creation_options.max_restarts, + actor_creation_options.max_task_retries, actor_creation_options.dynamic_worker_options, actor_creation_options.max_concurrency, actor_creation_options.is_detached, actor_name, @@ -1444,7 +1444,7 @@ Status CoreWorker::RemovePlacementGroup(const PlacementGroupID &placement_group_ // Synchronously wait for placement group removal. RAY_UNUSED(gcs_client_->PlacementGroups().AsyncRemovePlacementGroup( placement_group_id, - [status_promise](Status status) { status_promise->set_value(status); })); + [status_promise](const Status &status) { status_promise->set_value(status); })); auto status_future = status_promise->get_future(); if (status_future.wait_for(std::chrono::seconds( RayConfig::instance().gcs_server_request_timeout_seconds())) != @@ -1459,6 +1459,24 @@ Status CoreWorker::RemovePlacementGroup(const PlacementGroupID &placement_group_ return status_future.get(); } +Status CoreWorker::WaitPlacementGroupReady(const PlacementGroupID &placement_group_id, + int timeout_seconds) { + std::shared_ptr> status_promise = + std::make_shared>(); + RAY_CHECK_OK(gcs_client_->PlacementGroups().AsyncWaitUntilReady( + placement_group_id, + [status_promise](const Status &status) { status_promise->set_value(status); })); + auto status_future = status_promise->get_future(); + if (status_future.wait_for(std::chrono::seconds(timeout_seconds)) != + std::future_status::ready) { + std::ostringstream stream; + stream << "There was timeout in waiting for placement group " << placement_group_id + << " creation."; + return Status::TimedOut(stream.str()); + } + return status_future.get(); +} + void CoreWorker::SubmitActorTask(const ActorID &actor_id, const RayFunction &function, const std::vector> &args, const TaskOptions &task_options, @@ -1486,6 +1504,7 @@ void CoreWorker::SubmitActorTask(const ActorID &actor_id, const RayFunction &fun required_resources, return_ids, std::make_pair(PlacementGroupID::Nil(), -1), true, /* placement_group_capture_child_tasks */ + "", /* debugger_breakpoint */ override_environment_variables); // NOTE: placement_group_capture_child_tasks and override_environment_variables will be // ignored in the actor because we should always follow the actor's option. @@ -1784,7 +1803,7 @@ Status CoreWorker::ExecuteTask(const TaskSpecification &task_spec, status = options_.task_execution_callback( task_type, task_spec.GetName(), func, task_spec.GetRequiredResources().GetResourceMap(), args, arg_reference_ids, - return_ids, return_objects); + return_ids, task_spec.GetDebuggerBreakpoint(), return_objects); absl::optional caller_address( options_.is_local_mode ? absl::optional() @@ -1833,7 +1852,7 @@ Status CoreWorker::ExecuteTask(const TaskSpecification &task_spec, if (!options_.is_local_mode) { SetCurrentTaskId(TaskID::Nil()); - worker_context_.ResetCurrentTask(task_spec); + worker_context_.ResetCurrentTask(); } { absl::MutexLock lock(&mutex_); diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index e419adfd1..5e2770b71 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -61,7 +61,7 @@ struct CoreWorkerOptions { const std::unordered_map &required_resources, const std::vector> &args, const std::vector &arg_reference_ids, - const std::vector &return_ids, + const std::vector &return_ids, const std::string &debugger_breakpoint, std::vector> *results)>; CoreWorkerOptions() @@ -198,7 +198,7 @@ struct CoreWorkerOptions { /// thread with a worker. You can obtain the worker ID via /// `CoreWorkerProcess::GetCoreWorker()->GetWorkerID()`. Currently a Java worker process /// starts multiple workers by default, but can be configured to start only 1 worker by -/// overwriting the internal config `num_workers_per_process_java`. +/// speicifying `num_java_workers_per_process` in the job config. /// /// If only 1 worker is started (either because the worker type is driver, or the /// `num_workers` in `CoreWorkerOptions` is set to 1), all threads will be automatically @@ -632,12 +632,15 @@ class CoreWorker : public rpc::CoreWorkerServiceHandler { /// \param[in] max_retires max number of retry when the task fails. /// \param[in] placement_options placement group options. /// \param[in] placement_group_capture_child_tasks whether or not the submitted task + /// \param[in] debugger_breakpoint breakpoint to drop into for the debugger after this + /// task starts executing, or "" if we do not want to drop into the debugger. /// should capture parent's placement group implicilty. void SubmitTask(const RayFunction &function, const std::vector> &args, const TaskOptions &task_options, std::vector *return_ids, int max_retries, BundleID placement_options, - bool placement_group_capture_child_tasks); + bool placement_group_capture_child_tasks, + const std::string &debugger_breakpoint); /// Create an actor. /// @@ -675,6 +678,16 @@ class CoreWorker : public rpc::CoreWorkerServiceHandler { /// NotFound if placement group is already removed or doesn't exist. Status RemovePlacementGroup(const PlacementGroupID &placement_group_id); + /// Wait for a placement group until ready asynchronously. + /// Returns once the placement group is created or the timeout expires. + /// + /// \param placement_group The id of a placement group to wait for. + /// \param timeout_seconds Timeout in seconds. + /// \return Status OK if the placement group is created. TimedOut if request to GCS + /// server times out. NotFound if placement group is already removed or doesn't exist. + Status WaitPlacementGroupReady(const PlacementGroupID &placement_group_id, + int timeout_seconds); + /// Submit an actor task. /// /// \param[in] caller_id ID of the task submitter. diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc index ee8c76a29..b7fb72310 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc @@ -110,7 +110,7 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeInitialize( const std::unordered_map &required_resources, const std::vector> &args, const std::vector &arg_reference_ids, - const std::vector &return_ids, + const std::vector &return_ids, const std::string &debugger_breakpoint, std::vector> *results) { JNIEnv *env = GetJNIEnv(); RAY_CHECK(java_task_executor); diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc index 9115945d2..5470f70fb 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc @@ -222,7 +222,9 @@ JNIEXPORT jobject JNICALL Java_io_ray_runtime_task_NativeTaskSubmitter_nativeSub ray_function, task_args, task_options, &return_ids, /*max_retries=*/0, /*placement_options=*/ - std::pair(ray::PlacementGroupID::Nil(), 0), true); + std::pair(ray::PlacementGroupID::Nil(), 0), + /*placement_group_capture_child_tasks=*/true, + /*debugger_breakpoint*/ ""); // This is to avoid creating an empty java list and boost performance. if (return_ids.empty()) { @@ -293,6 +295,19 @@ Java_io_ray_runtime_task_NativeTaskSubmitter_nativeRemovePlacementGroup( THROW_EXCEPTION_AND_RETURN_IF_NOT_OK(env, status, (void)0); } +JNIEXPORT jboolean JNICALL +Java_io_ray_runtime_task_NativeTaskSubmitter_nativeWaitPlacementGroupReady( + JNIEnv *env, jclass p, jbyteArray placement_group_id_bytes, jint timeout_seconds) { + const auto placement_group_id = + JavaByteArrayToId(env, placement_group_id_bytes); + auto status = ray::CoreWorkerProcess::GetCoreWorker().WaitPlacementGroupReady( + placement_group_id, timeout_seconds); + if (status.IsNotFound()) { + env->ThrowNew(java_ray_exception_class, status.message().c_str()); + } + return status.ok(); +} + #ifdef __cplusplus } #endif diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.h b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.h index 33a46806e..8ea517b60 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.h +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.h @@ -71,6 +71,17 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_task_NativeTaskSubmitter_nativeRemovePlacementGroup(JNIEnv *, jclass, jbyteArray); +/* + * Class: io_ray_runtime_task_NativeTaskSubmitter + * Method: nativeWaitPlacementGroupReady + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_io_ray_runtime_task_NativeTaskSubmitter__nativeWaitPlacementGroupReady(JNIEnv *, + jclass, + jbyteArray, + jint); + #ifdef __cplusplus } #endif diff --git a/src/ray/core_worker/object_recovery_manager.cc b/src/ray/core_worker/object_recovery_manager.cc index 48346a8aa..f43b9e37d 100644 --- a/src/ray/core_worker/object_recovery_manager.cc +++ b/src/ray/core_worker/object_recovery_manager.cc @@ -18,16 +18,23 @@ namespace ray { -Status ObjectRecoveryManager::RecoverObject(const ObjectID &object_id) { +bool ObjectRecoveryManager::RecoverObject(const ObjectID &object_id) { // Check the ReferenceCounter to see if there is a location for the object. + bool owned_by_us = false; NodeID pinned_at; - bool spilled; - bool owned_by_us = - reference_counter_->IsPlasmaObjectPinnedOrSpilled(object_id, &pinned_at, &spilled); + bool spilled = false; + bool ref_exists = reference_counter_->IsPlasmaObjectPinnedOrSpilled( + object_id, &owned_by_us, &pinned_at, &spilled); + if (!ref_exists) { + // References that have gone out of scope cannot be recovered. + return false; + } + if (!owned_by_us) { - return Status::Invalid( - "Object reference no longer exists or is not owned by us. Either lineage pinning " - "is disabled or this object ID is borrowed."); + RAY_LOG(INFO) << "Reconstruction for borrowed objects (" << object_id + << ") is not supported"; + reconstruction_failure_callback_(object_id, /*pin_object=*/false); + return true; } bool already_pending_recovery = true; @@ -49,7 +56,7 @@ Status ObjectRecoveryManager::RecoverObject(const ObjectID &object_id) { RAY_LOG(INFO) << "Recovery complete for object " << object_id; }); // Lookup the object in the GCS to find another copy. - RAY_RETURN_NOT_OK(object_lookup_( + RAY_CHECK_OK(object_lookup_( object_id, [this](const ObjectID &object_id, const std::vector &locations) { PinOrReconstructObject(object_id, locations); @@ -57,7 +64,7 @@ Status ObjectRecoveryManager::RecoverObject(const ObjectID &object_id) { } else { RAY_LOG(DEBUG) << "Recovery already started for object " << object_id; } - return Status::OK(); + return true; } void ObjectRecoveryManager::PinOrReconstructObject( @@ -130,8 +137,8 @@ void ObjectRecoveryManager::ReconstructObject(const ObjectID &object_id) { if (status.ok()) { // Try to recover the task's dependencies. for (const auto &dep : task_deps) { - auto status = RecoverObject(dep); - if (!status.ok()) { + auto recovered = RecoverObject(dep); + if (!recovered) { RAY_LOG(INFO) << "Failed to reconstruct object " << dep << ": " << status.message(); // We do not pin the dependency because we may not be the owner. diff --git a/src/ray/core_worker/object_recovery_manager.h b/src/ray/core_worker/object_recovery_manager.h index c43f2de03..059fb8472 100644 --- a/src/ray/core_worker/object_recovery_manager.h +++ b/src/ray/core_worker/object_recovery_manager.h @@ -79,11 +79,12 @@ class ObjectRecoveryManager { /// plasma arguments to the task. The recovery operation will succeed once /// the task completes and stores a new value for its return object. /// - /// \return OK if recovery for the object has successfully started, Invalid - /// if the object is not recoverable because we do not own it. Note that the - /// Status::OK value only indicates that the recovery operation has started, - /// but does not guarantee that the recovery operation is successful. - Status RecoverObject(const ObjectID &object_id); + /// \return True if recovery for the object has successfully started, false + /// if the object is not recoverable because we do not have any metadata + /// about the object. If this returns true, then eventually recovery will + /// either succeed (a value will be put into the memory store) or fail (the + /// reconstruction failure callback will be called for this object). + bool RecoverObject(const ObjectID &object_id); private: /// Pin a new copy for a lost object from the given locations or, if that diff --git a/src/ray/core_worker/reference_count.cc b/src/ray/core_worker/reference_count.cc index a21350ef9..ef0168af7 100644 --- a/src/ray/core_worker/reference_count.cc +++ b/src/ray/core_worker/reference_count.cc @@ -543,16 +543,17 @@ void ReferenceCounter::UpdateObjectPinnedAtRaylet(const ObjectID &object_id, } bool ReferenceCounter::IsPlasmaObjectPinnedOrSpilled(const ObjectID &object_id, - NodeID *pinned_at, + bool *owned_by_us, NodeID *pinned_at, bool *spilled) const { absl::MutexLock lock(&mutex_); auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { if (it->second.owned_by_us) { + *owned_by_us = true; *spilled = it->second.spilled; *pinned_at = it->second.pinned_at_raylet_id.value_or(NodeID::Nil()); - return true; } + return true; } return false; } diff --git a/src/ray/core_worker/reference_count.h b/src/ray/core_worker/reference_count.h index bd950dae6..03bffa5fe 100644 --- a/src/ray/core_worker/reference_count.h +++ b/src/ray/core_worker/reference_count.h @@ -327,14 +327,15 @@ class ReferenceCounter : public ReferenceCounterInterface { /// available to fetch. /// /// \param[in] object_id The object to check. + /// \param[out] owned_by_us Whether this object is owned by us. The pinned_at + /// and spilled out-parameters are set if this is true. /// \param[out] pinned_at The node ID of the raylet at which this object is /// \param[out] spilled Whether this object has been spilled. /// pinned. Set to nil if the object is not pinned. - /// \return True if the object exists and is owned by us, false otherwise. We - /// return false here because a borrower should not know the pinned location - /// for an object. - bool IsPlasmaObjectPinnedOrSpilled(const ObjectID &object_id, NodeID *pinned_at, - bool *spilled) const LOCKS_EXCLUDED(mutex_); + /// \return True if the reference exists, false otherwise. + bool IsPlasmaObjectPinnedOrSpilled(const ObjectID &object_id, bool *owned_by_us, + NodeID *pinned_at, bool *spilled) const + LOCKS_EXCLUDED(mutex_); /// Get and reset the objects that were pinned on the given node. This /// method should be called upon a node failure, to determine which plasma diff --git a/src/ray/core_worker/reference_count_test.cc b/src/ray/core_worker/reference_count_test.cc index 46291943e..8362b2d21 100644 --- a/src/ray/core_worker/reference_count_test.cc +++ b/src/ray/core_worker/reference_count_test.cc @@ -1986,23 +1986,28 @@ TEST_F(ReferenceCountLineageEnabledTest, TestPlasmaLocation) { ObjectID borrowed_id = ObjectID::FromRandom(); rc->AddLocalReference(borrowed_id, ""); + bool owned_by_us; NodeID pinned_at; bool spilled; - ASSERT_FALSE(rc->IsPlasmaObjectPinnedOrSpilled(borrowed_id, &pinned_at, &spilled)); + ASSERT_TRUE( + rc->IsPlasmaObjectPinnedOrSpilled(borrowed_id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_FALSE(owned_by_us); ObjectID id = ObjectID::FromRandom(); NodeID node_id = NodeID::FromRandom(); rc->AddOwnedObject(id, {}, rpc::Address(), "", 0, true); rc->AddLocalReference(id, ""); ASSERT_TRUE(rc->SetDeleteCallback(id, callback)); - ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_TRUE(owned_by_us); ASSERT_TRUE(pinned_at.IsNil()); rc->UpdateObjectPinnedAtRaylet(id, node_id); - ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_TRUE(owned_by_us); ASSERT_FALSE(pinned_at.IsNil()); rc->RemoveLocalReference(id, nullptr); - ASSERT_FALSE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_FALSE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); ASSERT_TRUE(deleted->count(id) > 0); deleted->clear(); @@ -2013,7 +2018,8 @@ TEST_F(ReferenceCountLineageEnabledTest, TestPlasmaLocation) { auto objects = rc->ResetObjectsOnRemovedNode(node_id); ASSERT_EQ(objects.size(), 1); ASSERT_EQ(objects[0], id); - ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_TRUE(owned_by_us); ASSERT_TRUE(pinned_at.IsNil()); ASSERT_TRUE(deleted->count(id) > 0); deleted->clear(); @@ -2035,9 +2041,11 @@ TEST_F(ReferenceCountTest, TestFree) { ASSERT_FALSE(rc->SetDeleteCallback(id, callback)); ASSERT_EQ(deleted->count(id), 0); rc->UpdateObjectPinnedAtRaylet(id, node_id); + bool owned_by_us; NodeID pinned_at; bool spilled; - ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_TRUE(owned_by_us); ASSERT_TRUE(pinned_at.IsNil()); ASSERT_TRUE(rc->IsPlasmaObjectFreed(id)); rc->RemoveLocalReference(id, nullptr); @@ -2052,7 +2060,8 @@ TEST_F(ReferenceCountTest, TestFree) { rc->FreePlasmaObjects({id}); ASSERT_TRUE(rc->IsPlasmaObjectFreed(id)); ASSERT_TRUE(deleted->count(id) > 0); - ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &pinned_at, &spilled)); + ASSERT_TRUE(rc->IsPlasmaObjectPinnedOrSpilled(id, &owned_by_us, &pinned_at, &spilled)); + ASSERT_TRUE(owned_by_us); ASSERT_TRUE(pinned_at.IsNil()); rc->RemoveLocalReference(id, nullptr); ASSERT_FALSE(rc->IsPlasmaObjectFreed(id)); diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index 23d16890b..8591fa5df 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -257,7 +257,8 @@ void CoreWorkerTest::TestNormalTask(std::unordered_map &res TaskOptions options; std::vector return_ids; driver.SubmitTask(func, args, options, &return_ids, /*max_retries=*/0, - std::make_pair(PlacementGroupID::Nil(), -1), true); + std::make_pair(PlacementGroupID::Nil(), -1), true, + /*debugger_breakpoint=*/""); ASSERT_EQ(return_ids.size(), 1); @@ -533,7 +534,7 @@ TEST_F(ZeroNodeTest, TestTaskSpecPerf) { builder.SetCommonTaskSpec(RandomTaskId(), options.name, function.GetLanguage(), function.GetFunctionDescriptor(), job_id, RandomTaskId(), 0, RandomTaskId(), address, num_returns, resources, resources, - std::make_pair(PlacementGroupID::Nil(), -1), true); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); // Set task arguments. for (const auto &arg : args) { builder.AddArg(*arg); @@ -619,8 +620,9 @@ TEST_F(ZeroNodeTest, TestWorkerContext) { TaskSpecification task_spec; size_t num_returns = 3; + task_spec.GetMutableMessage().set_job_id(job_id.Binary()); task_spec.GetMutableMessage().set_num_returns(num_returns); - context.ResetCurrentTask(task_spec); + context.ResetCurrentTask(); context.SetCurrentTask(task_spec); ASSERT_EQ(context.GetCurrentTaskID(), task_spec.TaskId()); diff --git a/src/ray/core_worker/test/direct_task_transport_test.cc b/src/ray/core_worker/test/direct_task_transport_test.cc index a6056d45a..27af163b2 100644 --- a/src/ray/core_worker/test/direct_task_transport_test.cc +++ b/src/ray/core_worker/test/direct_task_transport_test.cc @@ -328,7 +328,7 @@ TaskSpecification BuildTaskSpec(const std::unordered_map &r builder.SetCommonTaskSpec(TaskID::Nil(), "dummy_task", Language::PYTHON, function_descriptor, JobID::Nil(), TaskID::Nil(), 0, TaskID::Nil(), empty_address, 1, resources, resources, - std::make_pair(PlacementGroupID::Nil(), -1), true); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); return builder.Build(); } diff --git a/src/ray/core_worker/test/mock_worker.cc b/src/ray/core_worker/test/mock_worker.cc index 47f111dfd..4439519bb 100644 --- a/src/ray/core_worker/test/mock_worker.cc +++ b/src/ray/core_worker/test/mock_worker.cc @@ -46,7 +46,7 @@ class MockWorker { options.node_manager_port = node_manager_port; options.raylet_ip_address = "127.0.0.1"; options.task_execution_callback = - std::bind(&MockWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8); + std::bind(&MockWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8, _9); options.ref_counting_enabled = true; options.num_workers = 1; options.metrics_agent_port = -1; @@ -62,6 +62,7 @@ class MockWorker { const std::vector> &args, const std::vector &arg_reference_ids, const std::vector &return_ids, + const std::string &debugger_breakpoint, std::vector> *results) { // Note that this doesn't include dummy object id. const ray::FunctionDescriptor function_descriptor = diff --git a/src/ray/core_worker/test/object_recovery_manager_test.cc b/src/ray/core_worker/test/object_recovery_manager_test.cc index da29ab908..a8d61599b 100644 --- a/src/ray/core_worker/test/object_recovery_manager_test.cc +++ b/src/ray/core_worker/test/object_recovery_manager_test.cc @@ -132,8 +132,6 @@ class ObjectRecoveryManagerTest : public ::testing::Test { std::make_shared(metadata, meta.size()); auto data = RayObject(nullptr, meta_buffer, std::vector()); RAY_CHECK(memory_store_->Put(data, object_id)); - - ref_counter_->UpdateObjectPinnedAtRaylet(object_id, local_raylet_id_); }, /*lineage_reconstruction_enabled=*/true) {} @@ -149,13 +147,27 @@ class ObjectRecoveryManagerTest : public ::testing::Test { }; TEST_F(ObjectRecoveryManagerTest, TestNoReconstruction) { + // Lineage recording disabled. ObjectID object_id = ObjectID::FromRandom(); ref_counter_->AddOwnedObject(object_id, {}, rpc::Address(), "", 0, true); - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); ASSERT_TRUE(failed_reconstructions_.empty()); ASSERT_TRUE(object_directory_->Flush() == 1); ASSERT_TRUE(failed_reconstructions_.count(object_id) == 1); ASSERT_EQ(task_resubmitter_->num_tasks_resubmitted, 0); + + // Borrowed object. + object_id = ObjectID::FromRandom(); + ref_counter_->AddLocalReference(object_id, ""); + ASSERT_TRUE(manager_.RecoverObject(object_id)); + ASSERT_TRUE(failed_reconstructions_.count(object_id) == 1); + ASSERT_EQ(task_resubmitter_->num_tasks_resubmitted, 0); + + // Ref went out of scope. + object_id = ObjectID::FromRandom(); + ASSERT_FALSE(manager_.RecoverObject(object_id)); + ASSERT_TRUE(failed_reconstructions_.count(object_id) == 0); + ASSERT_EQ(task_resubmitter_->num_tasks_resubmitted, 0); } TEST_F(ObjectRecoveryManagerTest, TestPinNewCopy) { @@ -164,7 +176,7 @@ TEST_F(ObjectRecoveryManagerTest, TestPinNewCopy) { std::vector addresses({rpc::Address()}); object_directory_->SetLocations(object_id, addresses); - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); ASSERT_TRUE(object_directory_->Flush() == 1); ASSERT_TRUE(raylet_client_->Flush() == 1); ASSERT_TRUE(failed_reconstructions_.empty()); @@ -176,7 +188,7 @@ TEST_F(ObjectRecoveryManagerTest, TestReconstruction) { ref_counter_->AddOwnedObject(object_id, {}, rpc::Address(), "", 0, true); task_resubmitter_->AddTask(object_id.TaskId(), {}); - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); ASSERT_TRUE(object_directory_->Flush() == 1); ASSERT_TRUE(failed_reconstructions_.empty()); @@ -188,24 +200,30 @@ TEST_F(ObjectRecoveryManagerTest, TestReconstructionSuppression) { ref_counter_->AddOwnedObject(object_id, {}, rpc::Address(), "", 0, true); ref_counter_->AddLocalReference(object_id, ""); - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); // A second attempt to recover the object will not trigger any more // callbacks. - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); + // A new copy of the object is pinned. + NodeID remote_node_id = NodeID::FromRandom(); + rpc::Address address; + address.set_raylet_id(remote_node_id.Binary()); + object_directory_->SetLocations(object_id, {address}); ASSERT_TRUE(object_directory_->Flush() == 1); - failed_reconstructions_.clear(); + ASSERT_TRUE(raylet_client_->Flush() == 1); - // The object has been marked as failed. Another attempt to recover the - // object will not trigger any callbacks. - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + // The object has been marked as failed but it is still pinned on the new + // node. Another attempt to recover the object will not trigger any + // callbacks. + ASSERT_TRUE(manager_.RecoverObject(object_id)); ASSERT_EQ(object_directory_->Flush(), 0); // The object is removed and can be recovered again. - auto objects = ref_counter_->ResetObjectsOnRemovedNode(local_raylet_id_); + auto objects = ref_counter_->ResetObjectsOnRemovedNode(remote_node_id); ASSERT_EQ(objects.size(), 1); ASSERT_EQ(objects[0], object_id); memory_store_->Delete(objects); - ASSERT_TRUE(manager_.RecoverObject(object_id).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_id)); ASSERT_TRUE(object_directory_->Flush() == 1); } @@ -220,7 +238,7 @@ TEST_F(ObjectRecoveryManagerTest, TestReconstructionChain) { object_ids.push_back(object_id); } - ASSERT_TRUE(manager_.RecoverObject(object_ids.back()).ok()); + ASSERT_TRUE(manager_.RecoverObject(object_ids.back())); for (int i = 0; i < 3; i++) { RAY_LOG(INFO) << i; ASSERT_EQ(object_directory_->Flush(), 1); diff --git a/src/ray/core_worker/transport/direct_actor_transport.cc b/src/ray/core_worker/transport/direct_actor_transport.cc index 19d911de1..e266b0d94 100644 --- a/src/ray/core_worker/transport/direct_actor_transport.cc +++ b/src/ray/core_worker/transport/direct_actor_transport.cc @@ -162,9 +162,9 @@ void CoreWorkerDirectActorTaskSubmitter::ConnectActor(const ActorID &actor_id, // TODO(swang): This assumes that all replies from the previous incarnation // of the actor have been received. Fix this by setting an epoch for each // actor task, so we can ignore completed tasks from old epochs. - RAY_LOG(INFO) << "Resetting caller starts at for actor " << actor_id << " from " - << queue->second.caller_starts_at << " to " - << queue->second.next_task_reply_position; + RAY_LOG(DEBUG) << "Resetting caller starts at for actor " << actor_id << " from " + << queue->second.caller_starts_at << " to " + << queue->second.next_task_reply_position; queue->second.caller_starts_at = queue->second.next_task_reply_position; ResendOutOfOrderTasks(actor_id); diff --git a/src/ray/core_worker/transport/direct_task_transport.cc b/src/ray/core_worker/transport/direct_task_transport.cc index 1d4cd40ef..c343c8d0f 100644 --- a/src/ray/core_worker/transport/direct_task_transport.cc +++ b/src/ray/core_worker/transport/direct_task_transport.cc @@ -48,7 +48,7 @@ Status CoreWorkerDirectTaskSubmitter::SubmitTask(TaskSpecification task_spec) { RAY_CHECK_OK(actor_creator_->AsyncCreateActor( task_spec, [this, actor_id, task_id](Status status) { if (status.ok()) { - RAY_LOG(INFO) << "Created actor, actor id = " << actor_id; + RAY_LOG(DEBUG) << "Created actor, actor id = " << actor_id; task_finisher_->CompletePendingTask(task_id, rpc::PushTaskReply(), rpc::Address()); } else { diff --git a/src/ray/gcs/accessor.h b/src/ray/gcs/accessor.h index a78c3c4cc..655c47aa7 100644 --- a/src/ray/gcs/accessor.h +++ b/src/ray/gcs/accessor.h @@ -488,6 +488,93 @@ class NodeInfoAccessor { /// \return Whether the node is removed. virtual bool IsRemoved(const NodeID &node_id) const = 0; + /// Report heartbeat of a node to GCS asynchronously. + /// + /// \param data_ptr The heartbeat that will be reported to GCS. + /// \param callback Callback that will be called after report finishes. + /// \return Status + // TODO(micafan) NodeStateAccessor will call this method to report heartbeat. + virtual Status AsyncReportHeartbeat( + const std::shared_ptr &data_ptr, + const StatusCallback &callback) = 0; + + /// Report resource usage of a node to GCS asynchronously. + /// + /// \param data_ptr The data that will be reported to GCS. + /// \param callback Callback that will be called after report finishes. + /// \return Status + virtual Status AsyncReportResourceUsage( + const std::shared_ptr &data_ptr, + const StatusCallback &callback) = 0; + + /// Resend resource usage when GCS restarts from a failure. + virtual void AsyncReReportResourceUsage() = 0; + + /// Return resources in last report. Used by light heartbeat. + std::shared_ptr &GetLastResourceUsage() { + return last_resource_usage_; + } + + /// Get newest resource usage of all nodes from GCS asynchronously. + /// + /// \param callback Callback that will be called after lookup finishes. + /// \return Status + virtual Status AsyncGetAllResourceUsage( + const ItemCallback &callback) = 0; + + /// Subscribe batched state of all nodes from GCS. + /// + /// \param subscribe Callback that will be called each time when batch resource usage is + /// updated. + /// \param done Callback that will be called when subscription is complete. + /// \return Status + virtual Status AsyncSubscribeBatchedResourceUsage( + const ItemCallback &subscribe, + const StatusCallback &done) = 0; + + /// Reestablish subscription. + /// This should be called when GCS server restarts from a failure. + /// PubSub server restart will cause GCS server restart. In this case, we need to + /// resubscribe from PubSub server, otherwise we only need to fetch data from GCS + /// server. + /// + /// \param is_pubsub_server_restarted Whether pubsub server is restarted. + virtual void AsyncResubscribe(bool is_pubsub_server_restarted) = 0; + + /// Set the internal config string that will be used by all nodes started in the + /// cluster. + /// + /// \param config Map of config options + /// \return Status + virtual Status AsyncSetInternalConfig( + std::unordered_map &config) = 0; + + /// Get the internal config string from GCS. + /// + /// \param callback Processes a map of config options + /// \return Status + virtual Status AsyncGetInternalConfig( + const OptionalItemCallback> + &callback) = 0; + + protected: + NodeInfoAccessor() = default; + + private: + /// Cache which stores resource usage in last report used to check if they are changed. + /// Used by light resource usage report. + std::shared_ptr last_resource_usage_ = + std::make_shared(); +}; + +/// \class NodeResourceInfoAccessor +/// `NodeResourceInfoAccessor` is a sub-interface of `GcsClient`. +/// This class includes all the methods that are related to accessing +/// node resource information in the GCS. +class NodeResourceInfoAccessor { + public: + virtual ~NodeResourceInfoAccessor() = default; + // TODO(micafan) Define ResourceMap in GCS proto. typedef std::unordered_map> ResourceMap; @@ -533,42 +620,6 @@ class NodeInfoAccessor { const ItemCallback &subscribe, const StatusCallback &done) = 0; - /// Report heartbeat of a node to GCS asynchronously. - /// - /// \param data_ptr The heartbeat that will be reported to GCS. - /// \param callback Callback that will be called after report finishes. - /// \return Status - // TODO(micafan) NodeStateAccessor will call this method to report heartbeat. - virtual Status AsyncReportHeartbeat( - const std::shared_ptr &data_ptr, - const StatusCallback &callback) = 0; - - /// Resend heartbeat when GCS restarts from a failure. - virtual void AsyncReReportHeartbeat() = 0; - - /// Return resources in last report. Used by light heartbeat. - std::shared_ptr &GetLastHeartbeatResources() { - return last_heartbeat_resources_; - } - - /// Get newest heartbeat of all nodes from GCS asynchronously. Only used when light - /// heartbeat enabled. - /// - /// \param callback Callback that will be called after lookup finishes. - /// \return Status - virtual Status AsyncGetAllHeartbeat( - const ItemCallback &callback) = 0; - - /// Subscribe batched state of all nodes from GCS. - /// - /// \param subscribe Callback that will be called each time when batch heartbeat is - /// updated. - /// \param done Callback that will be called when subscription is complete. - /// \return Status - virtual Status AsyncSubscribeBatchHeartbeat( - const ItemCallback &subscribe, - const StatusCallback &done) = 0; - /// Reestablish subscription. /// This should be called when GCS server restarts from a failure. /// PubSub server restart will cause GCS server restart. In this case, we need to @@ -578,30 +629,8 @@ class NodeInfoAccessor { /// \param is_pubsub_server_restarted Whether pubsub server is restarted. virtual void AsyncResubscribe(bool is_pubsub_server_restarted) = 0; - /// Set the internal config string that will be used by all nodes started in the - /// cluster. - /// - /// \param config Map of config options - /// \return Status - virtual Status AsyncSetInternalConfig( - std::unordered_map &config) = 0; - - /// Get the internal config string from GCS. - /// - /// \param callback Processes a map of config options - /// \return Status - virtual Status AsyncGetInternalConfig( - const OptionalItemCallback> - &callback) = 0; - protected: - NodeInfoAccessor() = default; - - private: - /// Cache which stores resources in last heartbeat used to check if they are changed. - /// Used by light heartbeat. - std::shared_ptr last_heartbeat_resources_ = - std::make_shared(); + NodeResourceInfoAccessor() = default; }; /// \class ErrorInfoAccessor @@ -749,7 +778,7 @@ class PlacementGroupInfoAccessor { virtual Status AsyncGetAll( const MultiItemCallback &callback) = 0; - /// Remove a placement group to GCS synchronously. + /// Remove a placement group to GCS asynchronously. /// /// \param placement_group_id The id for the placement group to remove. /// \param callback Callback that will be called after the placement group is @@ -758,6 +787,14 @@ class PlacementGroupInfoAccessor { virtual Status AsyncRemovePlacementGroup(const PlacementGroupID &placement_group_id, const StatusCallback &callback) = 0; + /// Wait for a placement group until ready asynchronously. + /// + /// \param placement_group_id The id for the placement group to wait for until ready. + /// \param callback Callback that will be called after the placement group is created. + /// \return Status + virtual Status AsyncWaitUntilReady(const PlacementGroupID &placement_group_id, + const StatusCallback &callback) = 0; + protected: PlacementGroupInfoAccessor() = default; }; diff --git a/src/ray/gcs/gcs_client.h b/src/ray/gcs/gcs_client.h index d88389c46..5f804606e 100644 --- a/src/ray/gcs/gcs_client.h +++ b/src/ray/gcs/gcs_client.h @@ -107,6 +107,13 @@ class GcsClient : public std::enable_shared_from_this { return *node_accessor_; } + /// Get the sub-interface for accessing node resource information in GCS. + /// This function is thread safe. + NodeResourceInfoAccessor &NodeResources() { + RAY_CHECK(node_resource_accessor_ != nullptr); + return *node_resource_accessor_; + } + /// Get the sub-interface for accessing task information in GCS. /// This function is thread safe. TaskInfoAccessor &Tasks() { @@ -157,6 +164,7 @@ class GcsClient : public std::enable_shared_from_this { std::unique_ptr job_accessor_; std::unique_ptr object_accessor_; std::unique_ptr node_accessor_; + std::unique_ptr node_resource_accessor_; std::unique_ptr task_accessor_; std::unique_ptr error_accessor_; std::unique_ptr stats_accessor_; diff --git a/src/ray/gcs/gcs_client/global_state_accessor.cc b/src/ray/gcs/gcs_client/global_state_accessor.cc index a1162ced0..8d188ba07 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.cc +++ b/src/ray/gcs/gcs_client/global_state_accessor.cc @@ -128,7 +128,8 @@ std::string GlobalStateAccessor::GetNodeResourceInfo(const NodeID &node_id) { auto on_done = [&node_resource_map, &promise]( const Status &status, - const boost::optional &result) { + const boost::optional + &result) { RAY_CHECK_OK(status); if (result) { auto result_value = result.get(); @@ -138,7 +139,7 @@ std::string GlobalStateAccessor::GetNodeResourceInfo(const NodeID &node_id) { } promise.set_value(); }; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetResources(node_id, on_done)); + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncGetResources(node_id, on_done)); promise.get_future().get(); return node_resource_map.SerializeAsString(); } @@ -146,7 +147,7 @@ std::string GlobalStateAccessor::GetNodeResourceInfo(const NodeID &node_id) { std::vector GlobalStateAccessor::GetAllAvailableResources() { std::vector available_resources; std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetAllAvailableResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncGetAllAvailableResources( TransformForMultiItemCallback(available_resources, promise))); promise.get_future().get(); @@ -174,14 +175,14 @@ std::string GlobalStateAccessor::GetInternalConfig() { return config_proto.SerializeAsString(); } -std::unique_ptr GlobalStateAccessor::GetAllHeartbeat() { - std::unique_ptr heartbeat_batch_data; +std::unique_ptr GlobalStateAccessor::GetAllResourceUsage() { + std::unique_ptr resource_batch_data; std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetAllHeartbeat( - TransformForItemCallback(heartbeat_batch_data, - promise))); + RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetAllResourceUsage( + TransformForItemCallback(resource_batch_data, + promise))); promise.get_future().get(); - return heartbeat_batch_data; + return resource_batch_data; } std::vector GlobalStateAccessor::GetAllActorInfo() { diff --git a/src/ray/gcs/gcs_client/global_state_accessor.h b/src/ray/gcs/gcs_client/global_state_accessor.h index 01c3ebb12..0c5695780 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.h +++ b/src/ray/gcs/gcs_client/global_state_accessor.h @@ -99,13 +99,13 @@ class GlobalStateAccessor { /// and serialized as a string to allow multi-language support. std::string GetInternalConfig(); - /// Get newest heartbeat of all nodes from GCS Service. Only used when light - /// heartbeat enabled. + /// Get newest resource usage of all nodes from GCS Service. Only used when light + /// rerouce usage report enabled. /// - /// \return node heartbeat info. To support multi-language, we serialize each - /// HeartbeatTableData and return the serialized string. Where used, it needs to be + /// \return resource usage info. To support multi-language, we serialize each + /// data and return the serialized string. Where used, it needs to be /// deserialized with protobuf function. - std::unique_ptr GetAllHeartbeat(); + std::unique_ptr GetAllResourceUsage(); /// Get information of all actors from GCS Service. /// diff --git a/src/ray/gcs/gcs_client/service_based_accessor.cc b/src/ray/gcs/gcs_client/service_based_accessor.cc index 2abaf0783..cc2907c63 100644 --- a/src/ray/gcs/gcs_client/service_based_accessor.cc +++ b/src/ray/gcs/gcs_client/service_based_accessor.cc @@ -501,119 +501,13 @@ bool ServiceBasedNodeInfoAccessor::IsRemoved(const NodeID &node_id) const { return removed_nodes_.count(node_id) == 1; } -Status ServiceBasedNodeInfoAccessor::AsyncGetResources( - const NodeID &node_id, const OptionalItemCallback &callback) { - RAY_LOG(DEBUG) << "Getting node resources, node id = " << node_id; - rpc::GetResourcesRequest request; - request.set_node_id(node_id.Binary()); - client_impl_->GetGcsRpcClient().GetResources( - request, - [node_id, callback](const Status &status, const rpc::GetResourcesReply &reply) { - ResourceMap resource_map; - for (auto resource : reply.resources()) { - resource_map[resource.first] = - std::make_shared(resource.second); - } - callback(status, resource_map); - RAY_LOG(DEBUG) << "Finished getting node resources, status = " << status - << ", node id = " << node_id; - }); - return Status::OK(); -} - -Status ServiceBasedNodeInfoAccessor::AsyncGetAllAvailableResources( - const MultiItemCallback &callback) { - rpc::GetAllAvailableResourcesRequest request; - client_impl_->GetGcsRpcClient().GetAllAvailableResources( - request, - [callback](const Status &status, const rpc::GetAllAvailableResourcesReply &reply) { - std::vector result = - VectorFromProtobuf(reply.resources_list()); - callback(status, result); - RAY_LOG(DEBUG) << "Finished getting available resources of all nodes, status = " - << status; - }); - return Status::OK(); -} - -Status ServiceBasedNodeInfoAccessor::AsyncUpdateResources( - const NodeID &node_id, const ResourceMap &resources, const StatusCallback &callback) { - RAY_LOG(DEBUG) << "Updating node resources, node id = " << node_id; - rpc::UpdateResourcesRequest request; - request.set_node_id(node_id.Binary()); - for (auto &resource : resources) { - (*request.mutable_resources())[resource.first] = *resource.second; - } - - auto operation = [this, request, node_id, - callback](const SequencerDoneCallback &done_callback) { - client_impl_->GetGcsRpcClient().UpdateResources( - request, [node_id, callback, done_callback]( - const Status &status, const rpc::UpdateResourcesReply &reply) { - if (callback) { - callback(status); - } - RAY_LOG(DEBUG) << "Finished updating node resources, status = " << status - << ", node id = " << node_id; - done_callback(); - }); - }; - - sequencer_.Post(node_id, operation); - return Status::OK(); -} - -Status ServiceBasedNodeInfoAccessor::AsyncDeleteResources( - const NodeID &node_id, const std::vector &resource_names, - const StatusCallback &callback) { - RAY_LOG(DEBUG) << "Deleting node resources, node id = " << node_id; - rpc::DeleteResourcesRequest request; - request.set_node_id(node_id.Binary()); - for (auto &resource_name : resource_names) { - request.add_resource_name_list(resource_name); - } - - auto operation = [this, request, node_id, - callback](const SequencerDoneCallback &done_callback) { - client_impl_->GetGcsRpcClient().DeleteResources( - request, [node_id, callback, done_callback]( - const Status &status, const rpc::DeleteResourcesReply &reply) { - if (callback) { - callback(status); - } - RAY_LOG(DEBUG) << "Finished deleting node resources, status = " << status - << ", node id = " << node_id; - done_callback(); - }); - }; - - sequencer_.Post(node_id, operation); - return Status::OK(); -} - -Status ServiceBasedNodeInfoAccessor::AsyncSubscribeToResources( - const ItemCallback &subscribe, const StatusCallback &done) { - RAY_CHECK(subscribe != nullptr); - subscribe_resource_operation_ = [this, subscribe](const StatusCallback &done) { - auto on_subscribe = [subscribe](const std::string &id, const std::string &data) { - rpc::NodeResourceChange node_resource_change; - node_resource_change.ParseFromString(data); - subscribe(node_resource_change); - }; - return client_impl_->GetGcsPubSub().SubscribeAll(NODE_RESOURCE_CHANNEL, on_subscribe, - done); - }; - return subscribe_resource_operation_(done); -} - Status ServiceBasedNodeInfoAccessor::AsyncReportHeartbeat( const std::shared_ptr &data_ptr, const StatusCallback &callback) { - absl::MutexLock lock(&mutex_); - cached_heartbeat_.mutable_heartbeat()->CopyFrom(*data_ptr); + rpc::ReportHeartbeatRequest request; + request.mutable_heartbeat()->CopyFrom(*data_ptr); client_impl_->GetGcsRpcClient().ReportHeartbeat( - cached_heartbeat_, - [callback](const Status &status, const rpc::ReportHeartbeatReply &reply) { + request, [callback](const Status &status, const rpc::ReportHeartbeatReply &reply) { if (callback) { callback(status); } @@ -621,74 +515,90 @@ Status ServiceBasedNodeInfoAccessor::AsyncReportHeartbeat( return Status::OK(); } -void ServiceBasedNodeInfoAccessor::AsyncReReportHeartbeat() { +Status ServiceBasedNodeInfoAccessor::AsyncReportResourceUsage( + const std::shared_ptr &data_ptr, const StatusCallback &callback) { absl::MutexLock lock(&mutex_); - if (cached_heartbeat_.has_heartbeat()) { - RAY_LOG(INFO) << "Rereport heartbeat."; - FillHeartbeatRequest(cached_heartbeat_); - client_impl_->GetGcsRpcClient().ReportHeartbeat( - cached_heartbeat_, - [](const Status &status, const rpc::ReportHeartbeatReply &reply) {}); - } -} - -void ServiceBasedNodeInfoAccessor::FillHeartbeatRequest( - rpc::ReportHeartbeatRequest &heartbeat) { - if (RayConfig::instance().light_heartbeat_enabled()) { - SchedulingResources cached_resources = - SchedulingResources(*GetLastHeartbeatResources()); - - auto heartbeat_data = heartbeat.mutable_heartbeat(); - heartbeat_data->clear_resources_total(); - for (const auto &resource_pair : - cached_resources.GetTotalResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_total())[resource_pair.first] = - resource_pair.second; - } - - heartbeat_data->clear_resources_available(); - heartbeat_data->set_resources_available_changed(true); - for (const auto &resource_pair : - cached_resources.GetAvailableResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_available())[resource_pair.first] = - resource_pair.second; - } - - heartbeat_data->clear_resource_load(); - heartbeat_data->set_resource_load_changed(true); - for (const auto &resource_pair : - cached_resources.GetLoadResources().GetResourceMap()) { - (*heartbeat_data->mutable_resource_load())[resource_pair.first] = - resource_pair.second; - } - } -} - -Status ServiceBasedNodeInfoAccessor::AsyncGetAllHeartbeat( - const ItemCallback &callback) { - rpc::GetAllHeartbeatRequest request; - client_impl_->GetGcsRpcClient().GetAllHeartbeat( - request, [callback](const Status &status, const rpc::GetAllHeartbeatReply &reply) { - callback(reply.heartbeat_data()); - RAY_LOG(DEBUG) << "Finished getting heartbeat of all nodes, status = " << status; + cached_resource_usage_.mutable_resources()->CopyFrom(*data_ptr); + client_impl_->GetGcsRpcClient().ReportResourceUsage( + cached_resource_usage_, + [callback](const Status &status, const rpc::ReportResourceUsageReply &reply) { + if (callback) { + callback(status); + } }); return Status::OK(); } -Status ServiceBasedNodeInfoAccessor::AsyncSubscribeBatchHeartbeat( - const ItemCallback &subscribe, +void ServiceBasedNodeInfoAccessor::AsyncReReportResourceUsage() { + absl::MutexLock lock(&mutex_); + if (cached_resource_usage_.has_resources()) { + RAY_LOG(INFO) << "Rereport resource usage."; + FillResourceUsageRequest(cached_resource_usage_); + client_impl_->GetGcsRpcClient().ReportResourceUsage( + cached_resource_usage_, + [](const Status &status, const rpc::ReportResourceUsageReply &reply) {}); + } +} + +void ServiceBasedNodeInfoAccessor::FillResourceUsageRequest( + rpc::ReportResourceUsageRequest &resources) { + if (RayConfig::instance().light_report_resource_usage_enabled()) { + SchedulingResources cached_resources = SchedulingResources(*GetLastResourceUsage()); + + auto resources_data = resources.mutable_resources(); + resources_data->clear_resources_total(); + for (const auto &resource_pair : + cached_resources.GetTotalResources().GetResourceMap()) { + (*resources_data->mutable_resources_total())[resource_pair.first] = + resource_pair.second; + } + + resources_data->clear_resources_available(); + resources_data->set_resources_available_changed(true); + for (const auto &resource_pair : + cached_resources.GetAvailableResources().GetResourceMap()) { + (*resources_data->mutable_resources_available())[resource_pair.first] = + resource_pair.second; + } + + resources_data->clear_resource_load(); + resources_data->set_resource_load_changed(true); + for (const auto &resource_pair : + cached_resources.GetLoadResources().GetResourceMap()) { + (*resources_data->mutable_resource_load())[resource_pair.first] = + resource_pair.second; + } + } +} + +Status ServiceBasedNodeInfoAccessor::AsyncGetAllResourceUsage( + const ItemCallback &callback) { + rpc::GetAllResourceUsageRequest request; + client_impl_->GetGcsRpcClient().GetAllResourceUsage( + request, + [callback](const Status &status, const rpc::GetAllResourceUsageReply &reply) { + callback(reply.resource_usage_data()); + RAY_LOG(DEBUG) << "Finished getting resource usage of all nodes, status = " + << status; + }); + return Status::OK(); +} + +Status ServiceBasedNodeInfoAccessor::AsyncSubscribeBatchedResourceUsage( + const ItemCallback &subscribe, const StatusCallback &done) { RAY_CHECK(subscribe != nullptr); - subscribe_batch_heartbeat_operation_ = [this, subscribe](const StatusCallback &done) { + subscribe_batch_resource_usage_operation_ = [this, + subscribe](const StatusCallback &done) { auto on_subscribe = [subscribe](const std::string &id, const std::string &data) { - rpc::HeartbeatBatchTableData heartbeat_batch_table_data; - heartbeat_batch_table_data.ParseFromString(data); - subscribe(heartbeat_batch_table_data); + rpc::ResourceUsageBatchData resources_batch_data; + resources_batch_data.ParseFromString(data); + subscribe(resources_batch_data); }; - return client_impl_->GetGcsPubSub().Subscribe(HEARTBEAT_BATCH_CHANNEL, "", + return client_impl_->GetGcsPubSub().Subscribe(RESOURCES_BATCH_CHANNEL, "", on_subscribe, done); }; - return subscribe_batch_heartbeat_operation_(done); + return subscribe_batch_resource_usage_operation_(done); } void ServiceBasedNodeInfoAccessor::HandleNotification(const GcsNodeInfo &node_info) { @@ -749,11 +659,8 @@ void ServiceBasedNodeInfoAccessor::AsyncResubscribe(bool is_pubsub_server_restar RAY_CHECK_OK(subscribe_node_operation_( [this](const Status &status) { fetch_node_data_operation_(nullptr); })); } - if (subscribe_resource_operation_ != nullptr) { - RAY_CHECK_OK(subscribe_resource_operation_(nullptr)); - } - if (subscribe_batch_heartbeat_operation_ != nullptr) { - RAY_CHECK_OK(subscribe_batch_heartbeat_operation_(nullptr)); + if (subscribe_batch_resource_usage_operation_ != nullptr) { + RAY_CHECK_OK(subscribe_batch_resource_usage_operation_(nullptr)); } } else { if (fetch_node_data_operation_ != nullptr) { @@ -799,6 +706,125 @@ Status ServiceBasedNodeInfoAccessor::AsyncGetInternalConfig( return Status::OK(); } +ServiceBasedNodeResourceInfoAccessor::ServiceBasedNodeResourceInfoAccessor( + ServiceBasedGcsClient *client_impl) + : client_impl_(client_impl) {} + +Status ServiceBasedNodeResourceInfoAccessor::AsyncGetResources( + const NodeID &node_id, const OptionalItemCallback &callback) { + RAY_LOG(DEBUG) << "Getting node resources, node id = " << node_id; + rpc::GetResourcesRequest request; + request.set_node_id(node_id.Binary()); + client_impl_->GetGcsRpcClient().GetResources( + request, + [node_id, callback](const Status &status, const rpc::GetResourcesReply &reply) { + ResourceMap resource_map; + for (auto resource : reply.resources()) { + resource_map[resource.first] = + std::make_shared(resource.second); + } + callback(status, resource_map); + RAY_LOG(DEBUG) << "Finished getting node resources, status = " << status + << ", node id = " << node_id; + }); + return Status::OK(); +} + +Status ServiceBasedNodeResourceInfoAccessor::AsyncGetAllAvailableResources( + const MultiItemCallback &callback) { + rpc::GetAllAvailableResourcesRequest request; + client_impl_->GetGcsRpcClient().GetAllAvailableResources( + request, + [callback](const Status &status, const rpc::GetAllAvailableResourcesReply &reply) { + std::vector result = + VectorFromProtobuf(reply.resources_list()); + callback(status, result); + RAY_LOG(DEBUG) << "Finished getting available resources of all nodes, status = " + << status; + }); + return Status::OK(); +} + +Status ServiceBasedNodeResourceInfoAccessor::AsyncUpdateResources( + const NodeID &node_id, const ResourceMap &resources, const StatusCallback &callback) { + RAY_LOG(DEBUG) << "Updating node resources, node id = " << node_id; + rpc::UpdateResourcesRequest request; + request.set_node_id(node_id.Binary()); + for (auto &resource : resources) { + (*request.mutable_resources())[resource.first] = *resource.second; + } + + auto operation = [this, request, node_id, + callback](const SequencerDoneCallback &done_callback) { + client_impl_->GetGcsRpcClient().UpdateResources( + request, [node_id, callback, done_callback]( + const Status &status, const rpc::UpdateResourcesReply &reply) { + if (callback) { + callback(status); + } + RAY_LOG(DEBUG) << "Finished updating node resources, status = " << status + << ", node id = " << node_id; + done_callback(); + }); + }; + + sequencer_.Post(node_id, operation); + return Status::OK(); +} + +Status ServiceBasedNodeResourceInfoAccessor::AsyncDeleteResources( + const NodeID &node_id, const std::vector &resource_names, + const StatusCallback &callback) { + RAY_LOG(DEBUG) << "Deleting node resources, node id = " << node_id; + rpc::DeleteResourcesRequest request; + request.set_node_id(node_id.Binary()); + for (auto &resource_name : resource_names) { + request.add_resource_name_list(resource_name); + } + + auto operation = [this, request, node_id, + callback](const SequencerDoneCallback &done_callback) { + client_impl_->GetGcsRpcClient().DeleteResources( + request, [node_id, callback, done_callback]( + const Status &status, const rpc::DeleteResourcesReply &reply) { + if (callback) { + callback(status); + } + RAY_LOG(DEBUG) << "Finished deleting node resources, status = " << status + << ", node id = " << node_id; + done_callback(); + }); + }; + + sequencer_.Post(node_id, operation); + return Status::OK(); +} + +Status ServiceBasedNodeResourceInfoAccessor::AsyncSubscribeToResources( + const ItemCallback &subscribe, const StatusCallback &done) { + RAY_CHECK(subscribe != nullptr); + subscribe_resource_operation_ = [this, subscribe](const StatusCallback &done) { + auto on_subscribe = [subscribe](const std::string &id, const std::string &data) { + rpc::NodeResourceChange node_resource_change; + node_resource_change.ParseFromString(data); + subscribe(node_resource_change); + }; + return client_impl_->GetGcsPubSub().SubscribeAll(NODE_RESOURCE_CHANNEL, on_subscribe, + done); + }; + return subscribe_resource_operation_(done); +} + +void ServiceBasedNodeResourceInfoAccessor::AsyncResubscribe( + bool is_pubsub_server_restarted) { + RAY_LOG(DEBUG) << "Reestablishing subscription for node resource info."; + // If the pub-sub server has also restarted, we need to resubscribe to the pub-sub + // server. + if (is_pubsub_server_restarted && subscribe_resource_operation_ != nullptr) { + RAY_CHECK_OK(subscribe_resource_operation_(nullptr)); + } +} + ServiceBasedTaskInfoAccessor::ServiceBasedTaskInfoAccessor( ServiceBasedGcsClient *client_impl) : client_impl_(client_impl) {} @@ -1510,5 +1536,23 @@ Status ServiceBasedPlacementGroupInfoAccessor::AsyncGetAll( return Status::OK(); } +Status ServiceBasedPlacementGroupInfoAccessor::AsyncWaitUntilReady( + const PlacementGroupID &placement_group_id, const StatusCallback &callback) { + RAY_LOG(DEBUG) << "Waiting for placement group until ready, placement group id = " + << placement_group_id; + rpc::WaitPlacementGroupUntilReadyRequest request; + request.set_placement_group_id(placement_group_id.Binary()); + client_impl_->GetGcsRpcClient().WaitPlacementGroupUntilReady( + request, + [placement_group_id, callback]( + const Status &status, const rpc::WaitPlacementGroupUntilReadyReply &reply) { + callback(status); + RAY_LOG(DEBUG) + << "Finished waiting placement group until ready, placement group id = " + << placement_group_id; + }); + return Status::OK(); +} + } // namespace gcs } // namespace ray diff --git a/src/ray/gcs/gcs_client/service_based_accessor.h b/src/ray/gcs/gcs_client/service_based_accessor.h index 1ad9da10f..f0e1f45bc 100644 --- a/src/ray/gcs/gcs_client/service_based_accessor.h +++ b/src/ray/gcs/gcs_client/service_based_accessor.h @@ -163,6 +163,78 @@ class ServiceBasedNodeInfoAccessor : public NodeInfoAccessor { bool IsRemoved(const NodeID &node_id) const override; + Status AsyncReportHeartbeat(const std::shared_ptr &data_ptr, + const StatusCallback &callback) override; + + Status AsyncReportResourceUsage(const std::shared_ptr &data_ptr, + const StatusCallback &callback) override; + + void AsyncReReportResourceUsage() override; + + /// Fill resource fields with cached resources. Used by light resource usage report. + void FillResourceUsageRequest(rpc::ReportResourceUsageRequest &resource_usage); + + Status AsyncGetAllResourceUsage( + const ItemCallback &callback) override; + + Status AsyncSubscribeBatchedResourceUsage( + const ItemCallback &subscribe, + const StatusCallback &done) override; + + void AsyncResubscribe(bool is_pubsub_server_restarted) override; + + Status AsyncSetInternalConfig( + std::unordered_map &config) override; + + Status AsyncGetInternalConfig( + const OptionalItemCallback> &callback) + override; + + private: + /// Save the subscribe operation in this function, so we can call it again when PubSub + /// server restarts from a failure. + SubscribeOperation subscribe_node_operation_; + SubscribeOperation subscribe_batch_resource_usage_operation_; + + /// Save the fetch data operation in this function, so we can call it again when GCS + /// server restarts from a failure. + FetchDataOperation fetch_node_data_operation_; + + // Mutex to protect the cached_resource_usage_ field. + absl::Mutex mutex_; + + /// Save the resource usage data, so we can resend it again when GCS server restarts + /// from a failure. + rpc::ReportResourceUsageRequest cached_resource_usage_ GUARDED_BY(mutex_); + + void HandleNotification(const GcsNodeInfo &node_info); + + ServiceBasedGcsClient *client_impl_; + + using NodeChangeCallback = + std::function; + + GcsNodeInfo local_node_info_; + NodeID local_node_id_; + + /// The callback to call when a new node is added or a node is removed. + NodeChangeCallback node_change_callback_{nullptr}; + + /// A cache for information about all nodes. + std::unordered_map node_cache_; + /// The set of removed nodes. + std::unordered_set removed_nodes_; +}; + +/// \class ServiceBasedNodeResourceInfoAccessor +/// ServiceBasedNodeResourceInfoAccessor is an implementation of +/// `NodeResourceInfoAccessor` that uses GCS Service as the backend. +class ServiceBasedNodeResourceInfoAccessor : public NodeResourceInfoAccessor { + public: + explicit ServiceBasedNodeResourceInfoAccessor(ServiceBasedGcsClient *client_impl); + + virtual ~ServiceBasedNodeResourceInfoAccessor() = default; + Status AsyncGetResources(const NodeID &node_id, const OptionalItemCallback &callback) override; @@ -179,67 +251,16 @@ class ServiceBasedNodeInfoAccessor : public NodeInfoAccessor { Status AsyncSubscribeToResources(const ItemCallback &subscribe, const StatusCallback &done) override; - Status AsyncReportHeartbeat(const std::shared_ptr &data_ptr, - const StatusCallback &callback) override; - - void AsyncReReportHeartbeat() override; - - /// Fill resource fields with cached resources. Used by light heartbeat. - void FillHeartbeatRequest(rpc::ReportHeartbeatRequest &heartbeat); - - Status AsyncGetAllHeartbeat( - const ItemCallback &callback) override; - - Status AsyncSubscribeBatchHeartbeat( - const ItemCallback &subscribe, - const StatusCallback &done) override; - void AsyncResubscribe(bool is_pubsub_server_restarted) override; - Status AsyncSetInternalConfig( - std::unordered_map &config) override; - - Status AsyncGetInternalConfig( - const OptionalItemCallback> &callback) - override; - private: /// Save the subscribe operation in this function, so we can call it again when PubSub /// server restarts from a failure. - SubscribeOperation subscribe_node_operation_; SubscribeOperation subscribe_resource_operation_; - SubscribeOperation subscribe_batch_heartbeat_operation_; - - /// Save the fetch data operation in this function, so we can call it again when GCS - /// server restarts from a failure. - FetchDataOperation fetch_node_data_operation_; - - // Mutex to protect the cached_heartbeat_ field. - absl::Mutex mutex_; - - /// Save the heartbeat data, so we can resend it again when GCS server restarts from a - /// failure. - rpc::ReportHeartbeatRequest cached_heartbeat_ GUARDED_BY(mutex_); - - void HandleNotification(const GcsNodeInfo &node_info); ServiceBasedGcsClient *client_impl_; - using NodeChangeCallback = - std::function; - - GcsNodeInfo local_node_info_; - NodeID local_node_id_; - Sequencer sequencer_; - - /// The callback to call when a new node is added or a node is removed. - NodeChangeCallback node_change_callback_{nullptr}; - - /// A cache for information about all nodes. - std::unordered_map node_cache_; - /// The set of removed nodes. - std::unordered_set removed_nodes_; }; /// \class ServiceBasedTaskInfoAccessor @@ -452,6 +473,9 @@ class ServiceBasedPlacementGroupInfoAccessor : public PlacementGroupInfoAccessor Status AsyncGetAll( const MultiItemCallback &callback) override; + Status AsyncWaitUntilReady(const PlacementGroupID &placement_group_id, + const StatusCallback &callback) override; + private: ServiceBasedGcsClient *client_impl_; }; diff --git a/src/ray/gcs/gcs_client/service_based_gcs_client.cc b/src/ray/gcs/gcs_client/service_based_gcs_client.cc index 552058f61..359f6cd81 100644 --- a/src/ray/gcs/gcs_client/service_based_gcs_client.cc +++ b/src/ray/gcs/gcs_client/service_based_gcs_client.cc @@ -57,6 +57,7 @@ Status ServiceBasedGcsClient::Connect(boost::asio::io_service &io_service) { job_accessor_->AsyncResubscribe(is_pubsub_server_restarted); actor_accessor_->AsyncResubscribe(is_pubsub_server_restarted); node_accessor_->AsyncResubscribe(is_pubsub_server_restarted); + node_resource_accessor_->AsyncResubscribe(is_pubsub_server_restarted); task_accessor_->AsyncResubscribe(is_pubsub_server_restarted); object_accessor_->AsyncResubscribe(is_pubsub_server_restarted); worker_accessor_->AsyncResubscribe(is_pubsub_server_restarted); @@ -70,6 +71,7 @@ Status ServiceBasedGcsClient::Connect(boost::asio::io_service &io_service) { job_accessor_.reset(new ServiceBasedJobInfoAccessor(this)); actor_accessor_.reset(new ServiceBasedActorInfoAccessor(this)); node_accessor_.reset(new ServiceBasedNodeInfoAccessor(this)); + node_resource_accessor_.reset(new ServiceBasedNodeResourceInfoAccessor(this)); task_accessor_.reset(new ServiceBasedTaskInfoAccessor(this)); object_accessor_.reset(new ServiceBasedObjectInfoAccessor(this)); stats_accessor_.reset(new ServiceBasedStatsInfoAccessor(this)); @@ -175,8 +177,8 @@ void ServiceBasedGcsClient::GcsServiceFailureDetected(rpc::GcsServiceFailureType // because we use the same Redis server for both GCS storage and pub-sub. So the // following flag is alway false. resubscribe_func_(false); - // Resend heartbeat after reconnected, needed by resource view in GCS. - node_accessor_->AsyncReReportHeartbeat(); + // Resend resource usage after reconnected, needed by resource view in GCS. + node_accessor_->AsyncReReportResourceUsage(); break; default: RAY_LOG(FATAL) << "Unsupported failure type: " << type; diff --git a/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc b/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc index 19533b087..0df0c7763 100644 --- a/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc +++ b/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc @@ -139,12 +139,12 @@ TEST_F(GlobalStateAccessorTest, TestNodeResourceTable) { RAY_CHECK_OK(gcs_client_->Nodes().AsyncRegister( *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); })); WaitReady(promise.get_future(), timeout_ms_); - ray::gcs::NodeInfoAccessor::ResourceMap resources; + ray::gcs::NodeResourceInfoAccessor::ResourceMap resources; rpc::ResourceTableData resource_table_data; resource_table_data.set_resource_capacity(static_cast(index + 1) + 0.1); resources[std::to_string(index)] = std::make_shared(resource_table_data); - RAY_IGNORE_EXPR(gcs_client_->Nodes().AsyncUpdateResources( + RAY_IGNORE_EXPR(gcs_client_->NodeResources().AsyncUpdateResources( node_id, resources, [](Status status) { RAY_CHECK(status.ok()); })); } auto node_table = global_state_->GetAllNodeInfo(); @@ -186,12 +186,11 @@ TEST_F(GlobalStateAccessorTest, TestInternalConfig) { } } -TEST_F(GlobalStateAccessorTest, TestGetAllHeartbeat) { - std::unique_ptr heartbeats = global_state_->GetAllHeartbeat(); - rpc::HeartbeatBatchTableData heartbeat_batch_data; - heartbeat_batch_data.ParseFromString(*heartbeats.get()); - - ASSERT_EQ(heartbeat_batch_data.batch_size(), 0); +TEST_F(GlobalStateAccessorTest, TestGetAllResourceUsage) { + std::unique_ptr resources = global_state_->GetAllResourceUsage(); + rpc::ResourceUsageBatchData resource_usage_batch_data; + resource_usage_batch_data.ParseFromString(*resources.get()); + ASSERT_EQ(resource_usage_batch_data.batch_size(), 0); auto node_table_data = Mocker::GenNodeInfo(); std::promise promise; @@ -201,59 +200,60 @@ TEST_F(GlobalStateAccessorTest, TestGetAllHeartbeat) { auto node_table = global_state_->GetAllNodeInfo(); ASSERT_EQ(node_table.size(), 1); - // Report heartbeat first time. + // Report resource usage first time. std::promise promise1; - auto heartbeat1 = std::make_shared(); - heartbeat1->set_node_id(node_table_data->node_id()); - RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportHeartbeat( - heartbeat1, [&promise1](Status status) { promise1.set_value(status.ok()); })); + auto resources1 = std::make_shared(); + resources1->set_node_id(node_table_data->node_id()); + RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportResourceUsage( + resources1, [&promise1](Status status) { promise1.set_value(status.ok()); })); WaitReady(promise1.get_future(), timeout_ms_); - heartbeats = global_state_->GetAllHeartbeat(); - heartbeat_batch_data.ParseFromString(*heartbeats.get()); - ASSERT_EQ(heartbeat_batch_data.batch_size(), 1); + resources = global_state_->GetAllResourceUsage(); + resource_usage_batch_data.ParseFromString(*resources.get()); + ASSERT_EQ(resource_usage_batch_data.batch_size(), 1); - // Report heartbeat with resources changed. + // Report changed resource usage. std::promise promise2; - auto heartbeat2 = std::make_shared(); + auto heartbeat2 = std::make_shared(); heartbeat2->set_node_id(node_table_data->node_id()); (*heartbeat2->mutable_resources_total())["CPU"] = 1; (*heartbeat2->mutable_resources_total())["GPU"] = 10; heartbeat2->set_resources_available_changed(true); (*heartbeat2->mutable_resources_available())["GPU"] = 5; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportHeartbeat( + RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportResourceUsage( heartbeat2, [&promise2](Status status) { promise2.set_value(status.ok()); })); WaitReady(promise2.get_future(), timeout_ms_); - heartbeats = global_state_->GetAllHeartbeat(); - heartbeat_batch_data.ParseFromString(*heartbeats.get()); - ASSERT_EQ(heartbeat_batch_data.batch_size(), 1); - auto heartbeat_data = heartbeat_batch_data.mutable_batch()->at(0); - ASSERT_EQ(heartbeat_data.resources_total_size(), 2); - ASSERT_EQ((*heartbeat_data.mutable_resources_total())["CPU"], 1.0); - ASSERT_EQ((*heartbeat_data.mutable_resources_total())["GPU"], 10.0); - ASSERT_EQ(heartbeat_data.resources_available_size(), 1); - ASSERT_EQ((*heartbeat_data.mutable_resources_available())["GPU"], 5.0); + resources = global_state_->GetAllResourceUsage(); + resource_usage_batch_data.ParseFromString(*resources.get()); + ASSERT_EQ(resource_usage_batch_data.batch_size(), 1); + auto resources_data = resource_usage_batch_data.mutable_batch()->at(0); + ASSERT_EQ(resources_data.resources_total_size(), 2); + ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0); + ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0); + ASSERT_EQ(resources_data.resources_available_size(), 1); + ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0); - // Report heartbeat with resources unchanged. (Only works with light heartbeat enabled) + // Report unchanged resource usage. (Only works with light resource usage report + // enabled) std::promise promise3; - auto heartbeat3 = std::make_shared(); + auto heartbeat3 = std::make_shared(); heartbeat3->set_node_id(node_table_data->node_id()); (*heartbeat3->mutable_resources_available())["CPU"] = 1; (*heartbeat3->mutable_resources_available())["GPU"] = 6; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportHeartbeat( + RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportResourceUsage( heartbeat3, [&promise3](Status status) { promise3.set_value(status.ok()); })); WaitReady(promise3.get_future(), timeout_ms_); - heartbeats = global_state_->GetAllHeartbeat(); - heartbeat_batch_data.ParseFromString(*heartbeats.get()); - ASSERT_EQ(heartbeat_batch_data.batch_size(), 1); - heartbeat_data = heartbeat_batch_data.mutable_batch()->at(0); - ASSERT_EQ(heartbeat_data.resources_total_size(), 2); - ASSERT_EQ((*heartbeat_data.mutable_resources_total())["CPU"], 1.0); - ASSERT_EQ((*heartbeat_data.mutable_resources_total())["GPU"], 10.0); - ASSERT_EQ(heartbeat_data.resources_available_size(), 1); - ASSERT_EQ((*heartbeat_data.mutable_resources_available())["GPU"], 5.0); + resources = global_state_->GetAllResourceUsage(); + resource_usage_batch_data.ParseFromString(*resources.get()); + ASSERT_EQ(resource_usage_batch_data.batch_size(), 1); + resources_data = resource_usage_batch_data.mutable_batch()->at(0); + ASSERT_EQ(resources_data.resources_total_size(), 2); + ASSERT_EQ((*resources_data.mutable_resources_total())["CPU"], 1.0); + ASSERT_EQ((*resources_data.mutable_resources_total())["GPU"], 10.0); + ASSERT_EQ(resources_data.resources_available_size(), 1); + ASSERT_EQ((*resources_data.mutable_resources_available())["GPU"], 5.0); } TEST_F(GlobalStateAccessorTest, TestProfileTable) { diff --git a/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc b/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc index 5b14f8f67..9df66dfff 100644 --- a/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc @@ -286,18 +286,19 @@ class ServiceBasedGcsClientTest : public ::testing::Test { bool SubscribeToResources(const gcs::ItemCallback &subscribe) { std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncSubscribeToResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncSubscribeToResources( subscribe, [&promise](Status status) { promise.set_value(status.ok()); })); return WaitReady(promise.get_future(), timeout_ms_); } - gcs::NodeInfoAccessor::ResourceMap GetResources(const NodeID &node_id) { - gcs::NodeInfoAccessor::ResourceMap resource_map; + gcs::NodeResourceInfoAccessor::ResourceMap GetResources(const NodeID &node_id) { + gcs::NodeResourceInfoAccessor::ResourceMap resource_map; std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetResources( - node_id, [&resource_map, &promise]( - Status status, - const boost::optional &result) { + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncGetResources( + node_id, + [&resource_map, &promise]( + Status status, + const boost::optional &result) { if (result) { resource_map.insert(result->begin(), result->end()); } @@ -309,11 +310,11 @@ class ServiceBasedGcsClientTest : public ::testing::Test { bool UpdateResources(const NodeID &node_id, const std::string &key) { std::promise promise; - gcs::NodeInfoAccessor::ResourceMap resource_map; + gcs::NodeResourceInfoAccessor::ResourceMap resource_map; auto resource = std::make_shared(); resource->set_resource_capacity(1.0); resource_map[key] = resource; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncUpdateResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncUpdateResources( node_id, resource_map, [&promise](Status status) { promise.set_value(status.ok()); })); return WaitReady(promise.get_future(), timeout_ms_); @@ -322,16 +323,16 @@ class ServiceBasedGcsClientTest : public ::testing::Test { bool DeleteResources(const NodeID &node_id, const std::vector &resource_names) { std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncDeleteResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncDeleteResources( node_id, resource_names, [&promise](Status status) { promise.set_value(status.ok()); })); return WaitReady(promise.get_future(), timeout_ms_); } - bool SubscribeBatchHeartbeat( - const gcs::ItemCallback &subscribe) { + bool SubscribeBatchResourceUsage( + const gcs::ItemCallback &subscribe) { std::promise promise; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncSubscribeBatchHeartbeat( + RAY_CHECK_OK(gcs_client_->Nodes().AsyncSubscribeBatchedResourceUsage( subscribe, [&promise](Status status) { promise.set_value(status.ok()); })); return WaitReady(promise.get_future(), timeout_ms_); } @@ -343,10 +344,17 @@ class ServiceBasedGcsClientTest : public ::testing::Test { return WaitReady(promise.get_future(), timeout_ms_); } + bool ReportResourceUsage(const std::shared_ptr resources) { + std::promise promise; + RAY_CHECK_OK(gcs_client_->Nodes().AsyncReportResourceUsage( + resources, [&promise](Status status) { promise.set_value(status.ok()); })); + return WaitReady(promise.get_future(), timeout_ms_); + } + std::vector GetAllAvailableResources() { std::promise promise; std::vector resources; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetAllAvailableResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncGetAllAvailableResources( [&resources, &promise](Status status, const std::vector &result) { EXPECT_TRUE(!result.empty()); @@ -715,80 +723,76 @@ TEST_F(ServiceBasedGcsClientTest, TestNodeResources) { ASSERT_TRUE(GetResources(node_id).empty()); } -TEST_F(ServiceBasedGcsClientTest, TestNodeHeartbeat) { +TEST_F(ServiceBasedGcsClientTest, TestNodeResourceUsage) { // Subscribe batched state of all nodes from GCS. - std::atomic heartbeat_batch_count(0); - auto on_subscribe = - [&heartbeat_batch_count](const gcs::HeartbeatBatchTableData &result) { - ++heartbeat_batch_count; - }; - ASSERT_TRUE(SubscribeBatchHeartbeat(on_subscribe)); + std::atomic resource_batch_count(0); + auto on_subscribe = [&resource_batch_count](const gcs::ResourceUsageBatchData &result) { + ++resource_batch_count; + }; + ASSERT_TRUE(SubscribeBatchResourceUsage(on_subscribe)); // Register node. auto node_info = Mocker::GenNodeInfo(); RAY_CHECK(RegisterNode(*node_info)); - // Report heartbeat of a node to GCS. + // Report resource usage of a node to GCS. NodeID node_id = NodeID::FromBinary(node_info->node_id()); - auto heartbeat = std::make_shared(); - heartbeat->set_node_id(node_id.Binary()); - // Set this flag because GCS won't publish unchanged heartbeat. - heartbeat->set_should_global_gc(true); - ASSERT_TRUE(ReportHeartbeat(heartbeat)); - WaitForExpectedCount(heartbeat_batch_count, 1); + auto resource = std::make_shared(); + resource->set_node_id(node_id.Binary()); + resource->set_should_global_gc(true); + ASSERT_TRUE(ReportResourceUsage(resource)); + WaitForExpectedCount(resource_batch_count, 1); } -TEST_F(ServiceBasedGcsClientTest, TestNodeHeartbeatWithLightHeartbeat) { +TEST_F(ServiceBasedGcsClientTest, TestNodeResourceUsageWithLightResourceUsageReport) { // Subscribe batched state of all nodes from GCS. - std::atomic heartbeat_batch_count(0); - auto on_subscribe = - [&heartbeat_batch_count](const gcs::HeartbeatBatchTableData &result) { - ++heartbeat_batch_count; - }; - ASSERT_TRUE(SubscribeBatchHeartbeat(on_subscribe)); + std::atomic resource_batch_count(0); + auto on_subscribe = [&resource_batch_count](const gcs::ResourceUsageBatchData &result) { + ++resource_batch_count; + }; + ASSERT_TRUE(SubscribeBatchResourceUsage(on_subscribe)); // Register node. auto node_info = Mocker::GenNodeInfo(); RAY_CHECK(RegisterNode(*node_info)); - // Report unchanged heartbeat of a node to GCS. + // Report unchanged resource usage of a node to GCS. NodeID node_id = NodeID::FromBinary(node_info->node_id()); - auto heartbeat = std::make_shared(); - heartbeat->set_node_id(node_id.Binary()); - ASSERT_TRUE(ReportHeartbeat(heartbeat)); - WaitForExpectedCount(heartbeat_batch_count, 0); + auto resource = std::make_shared(); + resource->set_node_id(node_id.Binary()); + ASSERT_TRUE(ReportResourceUsage(resource)); + WaitForExpectedCount(resource_batch_count, 0); - // Report changed heartbeat of a node to GCS. - auto heartbeat1 = std::make_shared(); - heartbeat1->set_node_id(node_id.Binary()); - heartbeat1->set_resources_available_changed(true); - ASSERT_TRUE(ReportHeartbeat(heartbeat1)); - WaitForExpectedCount(heartbeat_batch_count, 1); + // Report changed resource usage of a node to GCS. + auto resource1 = std::make_shared(); + resource1->set_node_id(node_id.Binary()); + resource1->set_resources_available_changed(true); + ASSERT_TRUE(ReportResourceUsage(resource1)); + WaitForExpectedCount(resource_batch_count, 1); } TEST_F(ServiceBasedGcsClientTest, TestGetAllAvailableResources) { // Subscribe batched state of all nodes from GCS. - std::atomic heartbeat_batch_count(0); - auto on_subscribe = - [&heartbeat_batch_count](const gcs::HeartbeatBatchTableData &result) { - ++heartbeat_batch_count; - }; - ASSERT_TRUE(SubscribeBatchHeartbeat(on_subscribe)); + std::atomic resource_batch_count(0); + auto on_subscribe = [&resource_batch_count](const gcs::ResourceUsageBatchData &result) { + ++resource_batch_count; + }; + ASSERT_TRUE(SubscribeBatchResourceUsage(on_subscribe)); // Register node. auto node_info = Mocker::GenNodeInfo(); RAY_CHECK(RegisterNode(*node_info)); - // Report heartbeat of a node to GCS. + // Report resource usage of a node to GCS. NodeID node_id = NodeID::FromBinary(node_info->node_id()); - auto heartbeat = std::make_shared(); - heartbeat->set_node_id(node_id.Binary()); + auto resource = std::make_shared(); + resource->set_node_id(node_id.Binary()); // Set this flag to indicate resources has changed. - heartbeat->set_resources_available_changed(true); - (*heartbeat->mutable_resources_available())["CPU"] = 1.0; - (*heartbeat->mutable_resources_available())["GPU"] = 10.0; - ASSERT_TRUE(ReportHeartbeat(heartbeat)); - WaitForExpectedCount(heartbeat_batch_count, 1); + resource->set_resources_available_changed(true); + (*resource->mutable_resources_available())["CPU"] = 1.0; + (*resource->mutable_resources_available())["GPU"] = 10.0; + ASSERT_TRUE(ReportResourceUsage(resource)); + WaitForExpectedCount(resource_batch_count, 1); // Assert get all available resources right. std::vector resources = GetAllAvailableResources(); @@ -798,28 +802,28 @@ TEST_F(ServiceBasedGcsClientTest, TestGetAllAvailableResources) { EXPECT_EQ((*resources[0].mutable_resources_available())["GPU"], 10.0); } -TEST_F(ServiceBasedGcsClientTest, TestGetAllAvailableResourcesWithLightHeartbeat) { +TEST_F(ServiceBasedGcsClientTest, + TestGetAllAvailableResourcesWithLightResourceUsageReport) { // Subscribe batched state of all nodes from GCS. - std::atomic heartbeat_batch_count(0); - auto on_subscribe = - [&heartbeat_batch_count](const gcs::HeartbeatBatchTableData &result) { - ++heartbeat_batch_count; - }; - ASSERT_TRUE(SubscribeBatchHeartbeat(on_subscribe)); + std::atomic resource_batch_count(0); + auto on_subscribe = [&resource_batch_count](const gcs::ResourceUsageBatchData &result) { + ++resource_batch_count; + }; + ASSERT_TRUE(SubscribeBatchResourceUsage(on_subscribe)); // Register node. auto node_info = Mocker::GenNodeInfo(); RAY_CHECK(RegisterNode(*node_info)); - // Report heartbeat of a node to GCS. + // Report resource usage of a node to GCS. NodeID node_id = NodeID::FromBinary(node_info->node_id()); - auto heartbeat = std::make_shared(); - heartbeat->set_node_id(node_id.Binary()); - heartbeat->set_resources_available_changed(true); - (*heartbeat->mutable_resources_available())["CPU"] = 1.0; - (*heartbeat->mutable_resources_available())["GPU"] = 10.0; - ASSERT_TRUE(ReportHeartbeat(heartbeat)); - WaitForExpectedCount(heartbeat_batch_count, 1); + auto resource = std::make_shared(); + resource->set_node_id(node_id.Binary()); + resource->set_resources_available_changed(true); + (*resource->mutable_resources_available())["CPU"] = 1.0; + (*resource->mutable_resources_available())["GPU"] = 10.0; + ASSERT_TRUE(ReportResourceUsage(resource)); + WaitForExpectedCount(resource_batch_count, 1); // Assert get all available resources right. std::vector resources = GetAllAvailableResources(); @@ -828,12 +832,12 @@ TEST_F(ServiceBasedGcsClientTest, TestGetAllAvailableResourcesWithLightHeartbeat EXPECT_EQ((*resources[0].mutable_resources_available())["CPU"], 1.0); EXPECT_EQ((*resources[0].mutable_resources_available())["GPU"], 10.0); - // Report unchanged heartbeat of a node to GCS. - auto heartbeat1 = std::make_shared(); - heartbeat1->set_node_id(node_id.Binary()); - (*heartbeat1->mutable_resources_available())["GPU"] = 8.0; - ASSERT_TRUE(ReportHeartbeat(heartbeat1)); - WaitForExpectedCount(heartbeat_batch_count, 1); + // Report unchanged resource usage of a node to GCS. + auto resource1 = std::make_shared(); + resource1->set_node_id(node_id.Binary()); + (*resource1->mutable_resources_available())["GPU"] = 8.0; + ASSERT_TRUE(ReportResourceUsage(resource1)); + WaitForExpectedCount(resource_batch_count, 1); // The value would remain unchanged. std::vector resources1 = GetAllAvailableResources(); @@ -1145,24 +1149,24 @@ TEST_F(ServiceBasedGcsClientTest, TestNodeTableResubscribe) { ASSERT_TRUE(SubscribeToResources(resource_subscribe)); // Subscribe batched state of all nodes from GCS. - std::atomic batch_heartbeat_count(0); - auto batch_heartbeat_subscribe = - [&batch_heartbeat_count](const rpc::HeartbeatBatchTableData &result) { - ++batch_heartbeat_count; + std::atomic batch_resource_usage_count(0); + auto batch_resource_usage_subscribe = + [&batch_resource_usage_count](const rpc::ResourceUsageBatchData &result) { + ++batch_resource_usage_count; }; - ASSERT_TRUE(SubscribeBatchHeartbeat(batch_heartbeat_subscribe)); + ASSERT_TRUE(SubscribeBatchResourceUsage(batch_resource_usage_subscribe)); auto node_info = Mocker::GenNodeInfo(1); ASSERT_TRUE(RegisterNode(*node_info)); NodeID node_id = NodeID::FromBinary(node_info->node_id()); std::string key = "CPU"; ASSERT_TRUE(UpdateResources(node_id, key)); - auto heartbeat = std::make_shared(); - heartbeat->set_node_id(node_info->node_id()); - // Set this flag because GCS won't publish unchanged heartbeat. - heartbeat->set_should_global_gc(true); - ASSERT_TRUE(ReportHeartbeat(heartbeat)); - WaitForExpectedCount(batch_heartbeat_count, 1); + auto resources = std::make_shared(); + resources->set_node_id(node_info->node_id()); + // Set this flag because GCS won't publish unchanged resources. + resources->set_should_global_gc(true); + ASSERT_TRUE(ReportResourceUsage(resources)); + WaitForExpectedCount(batch_resource_usage_count, 1); RestartGcsServer(); @@ -1170,12 +1174,12 @@ TEST_F(ServiceBasedGcsClientTest, TestNodeTableResubscribe) { ASSERT_TRUE(RegisterNode(*node_info)); node_id = NodeID::FromBinary(node_info->node_id()); ASSERT_TRUE(UpdateResources(node_id, key)); - heartbeat->set_node_id(node_info->node_id()); - ASSERT_TRUE(ReportHeartbeat(heartbeat)); + resources->set_node_id(node_info->node_id()); + ASSERT_TRUE(ReportResourceUsage(resources)); WaitForExpectedCount(node_change_count, 2); WaitForExpectedCount(resource_change_count, 2); - WaitForExpectedCount(batch_heartbeat_count, 2); + WaitForExpectedCount(batch_resource_usage_count, 2); } TEST_F(ServiceBasedGcsClientTest, TestTaskTableResubscribe) { diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index 8e2329c7b..aa420f463 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -28,7 +28,8 @@ GcsActorScheduler::GcsActorScheduler( std::shared_ptr gcs_pub_sub, std::function)> schedule_failure_handler, std::function)> schedule_success_handler, - LeaseClientFactoryFn lease_client_factory, rpc::ClientFactoryFn client_factory) + std::shared_ptr raylet_client_pool, + rpc::ClientFactoryFn client_factory) : io_context_(io_context), gcs_actor_table_(gcs_actor_table), gcs_node_manager_(gcs_node_manager), @@ -36,7 +37,7 @@ GcsActorScheduler::GcsActorScheduler( schedule_failure_handler_(std::move(schedule_failure_handler)), schedule_success_handler_(std::move(schedule_success_handler)), report_worker_backlog_(RayConfig::instance().report_worker_backlog()), - lease_client_factory_(std::move(lease_client_factory)), + raylet_client_pool_(raylet_client_pool), core_worker_clients_(client_factory) { RAY_CHECK(schedule_failure_handler_ != nullptr && schedule_success_handler_ != nullptr); } @@ -45,7 +46,18 @@ void GcsActorScheduler::Schedule(std::shared_ptr actor) { RAY_CHECK(actor->GetNodeID().IsNil() && actor->GetWorkerID().IsNil()); // Select a node to lease worker for the actor. - auto node = SelectNodeRandomly(); + std::shared_ptr node; + + // If an actor has resource requirements, we will try to schedule it on the same node as + // the owner if possible. + const auto &task_spec = actor->GetCreationTaskSpecification(); + if (!task_spec.GetRequiredResources().IsEmpty()) { + auto maybe_node = gcs_node_manager_.GetAliveNode(actor->GetOwnerNodeID()); + node = maybe_node.has_value() ? maybe_node.value() : SelectNodeRandomly(); + } else { + node = SelectNodeRandomly(); + } + if (node == nullptr) { // There are no available nodes to schedule the actor, so just trigger the failed // handler. @@ -118,10 +130,7 @@ std::vector GcsActorScheduler::CancelOnNode(const NodeID &node_id) { } } - // Remove the related remote lease client from remote_lease_clients_. - // There is no need to check in this place, because it is possible that there are no - // workers leased on this node. - remote_lease_clients_.erase(node_id); + raylet_client_pool_->Disconnect(node_id); return actor_ids; } @@ -286,7 +295,7 @@ void GcsActorScheduler::HandleWorkerLeasedReply( // node, and then try again on the new node. RAY_CHECK(!retry_at_raylet_address.raylet_id().empty()); auto spill_back_node_id = NodeID::FromBinary(retry_at_raylet_address.raylet_id()); - auto maybe_spill_back_node = gcs_node_manager_.GetNode(spill_back_node_id); + auto maybe_spill_back_node = gcs_node_manager_.GetAliveNode(spill_back_node_id); if (maybe_spill_back_node.has_value()) { auto spill_back_node = maybe_spill_back_node.value(); actor->UpdateAddress(retry_at_raylet_address); @@ -423,13 +432,7 @@ std::shared_ptr GcsActorScheduler::SelectNodeRandomly() const std::shared_ptr GcsActorScheduler::GetOrConnectLeaseClient( const rpc::Address &raylet_address) { - auto node_id = NodeID::FromBinary(raylet_address.raylet_id()); - auto iter = remote_lease_clients_.find(node_id); - if (iter == remote_lease_clients_.end()) { - auto lease_client = lease_client_factory_(raylet_address); - iter = remote_lease_clients_.emplace(node_id, std::move(lease_client)).first; - } - return iter->second; + return raylet_client_pool_->GetOrConnectByAddress(raylet_address); } } // namespace gcs diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index c1ebebda9..b59c4b2d4 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -26,6 +26,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/node_manager_client.h" +#include "ray/rpc/node_manager/node_manager_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "src/ray/protobuf/gcs_service.pb.h" @@ -33,9 +34,6 @@ namespace ray { namespace gcs { -using LeaseClientFactoryFn = - std::function(const rpc::Address &address)>; - class GcsActor; class GcsActorSchedulerInterface { @@ -91,8 +89,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { /// schedule actors. /// \param schedule_success_handler Invoked when actors are created on the worker /// successfully. - /// \param lease_client_factory Factory to create remote lease client, default factor - /// will be used if not set. + /// \param raylet_client_pool Raylet client pool to construct connections to raylets. /// \param client_factory Factory to create remote core worker client, default factor /// will be used if not set. explicit GcsActorScheduler( @@ -100,7 +97,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { const GcsNodeManager &gcs_node_manager, std::shared_ptr gcs_pub_sub, std::function)> schedule_failure_handler, std::function)> schedule_success_handler, - LeaseClientFactoryFn lease_client_factory = nullptr, + std::shared_ptr raylet_client_pool, rpc::ClientFactoryFn client_factory = nullptr); virtual ~GcsActorScheduler() = default; @@ -275,9 +272,6 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { absl::flat_hash_map>> node_to_workers_when_creating_; - /// The cached node clients which are used to communicate with raylet to lease workers. - absl::flat_hash_map> - remote_lease_clients_; /// Reference of GcsNodeManager. const GcsNodeManager &gcs_node_manager_; /// A publisher for publishing gcs messages. @@ -288,10 +282,10 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { std::function)> schedule_success_handler_; /// Whether or not to report the backlog of actors waiting to be scheduled. bool report_worker_backlog_; - /// Factory for producing new clients to request leases from remote nodes. - LeaseClientFactoryFn lease_client_factory_; /// The nodes which are releasing unused workers. absl::flat_hash_set nodes_of_releasing_unused_workers_; + /// The cached raylet clients used to communicate with raylet. + std::shared_ptr raylet_client_pool_; /// The cached core worker clients which are used to communicate with leased worker. rpc::CoreWorkerClientPool core_worker_clients_; }; diff --git a/src/ray/gcs/gcs_server/gcs_heartbeat_manager.cc b/src/ray/gcs/gcs_server/gcs_heartbeat_manager.cc new file mode 100644 index 000000000..b16383097 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_heartbeat_manager.cc @@ -0,0 +1,111 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/gcs/gcs_server/gcs_heartbeat_manager.h" +#include "ray/common/ray_config.h" +#include "ray/gcs/pb_util.h" +#include "src/ray/protobuf/gcs.pb.h" + +namespace ray { +namespace gcs { + +GcsHeartbeatManager::GcsHeartbeatManager( + boost::asio::io_service &io_service, + std::function on_node_death_callback) + : io_service_(io_service), + on_node_death_callback_(std::move(on_node_death_callback)), + num_heartbeats_timeout_(RayConfig::instance().num_heartbeats_timeout()), + detect_timer_(io_service) { + io_service_thread_.reset(new std::thread([this] { + /// The asio work to keep io_service_ alive. + boost::asio::io_service::work io_service_work_(io_service_); + io_service_.run(); + })); +} + +void GcsHeartbeatManager::Start() { + io_service_.post([this] { + if (!is_started_) { + Tick(); + is_started_ = true; + } + }); +} + +void GcsHeartbeatManager::Stop() { + io_service_.stop(); + if (io_service_thread_->joinable()) { + io_service_thread_->join(); + } +} + +void GcsHeartbeatManager::AddNode(const NodeID &node_id) { + io_service_.post( + [this, node_id] { heartbeats_.emplace(node_id, num_heartbeats_timeout_); }); +} + +void GcsHeartbeatManager::HandleReportHeartbeat( + const rpc::ReportHeartbeatRequest &request, rpc::ReportHeartbeatReply *reply, + rpc::SendReplyCallback send_reply_callback) { + NodeID node_id = NodeID::FromBinary(request.heartbeat().node_id()); + auto iter = heartbeats_.find(node_id); + if (iter == heartbeats_.end()) { + // Reply the raylet with an error so the raylet can crash itself. + GCS_RPC_SEND_REPLY(send_reply_callback, reply, + Status::Disconnected("Node has been dead")); + return; + } + + iter->second = num_heartbeats_timeout_; + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); +} + +/// A periodic timer that checks for timed out clients. +void GcsHeartbeatManager::Tick() { + DetectDeadNodes(); + ScheduleTick(); +} + +void GcsHeartbeatManager::DetectDeadNodes() { + for (auto it = heartbeats_.begin(); it != heartbeats_.end();) { + auto current = it++; + current->second = current->second - 1; + if (current->second == 0) { + auto node_id = current->first; + RAY_LOG(WARNING) << "Node timed out: " << node_id; + heartbeats_.erase(current); + if (on_node_death_callback_) { + on_node_death_callback_(node_id); + } + } + } +} + +void GcsHeartbeatManager::ScheduleTick() { + auto heartbeat_period = boost::posix_time::milliseconds( + RayConfig::instance().raylet_heartbeat_timeout_milliseconds()); + detect_timer_.expires_from_now(heartbeat_period); + detect_timer_.async_wait([this](const boost::system::error_code &error) { + if (error == boost::asio::error::operation_aborted) { + // `operation_aborted` is set when `detect_timer_` is canceled or destroyed. + // The Monitor lifetime may be short than the object who use it. (e.g. gcs_server) + return; + } + RAY_CHECK(!error) << "Checking heartbeat failed with error: " << error.message(); + Tick(); + }); +} + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_heartbeat_manager.h b/src/ray/gcs/gcs_server/gcs_heartbeat_manager.h new file mode 100644 index 000000000..580daa6f3 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_heartbeat_manager.h @@ -0,0 +1,89 @@ + +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/container/flat_hash_map.h" +#include "ray/common/id.h" +#include "ray/gcs/accessor.h" +#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/rpc/client_call.h" +#include "ray/rpc/gcs_server/gcs_rpc_server.h" +#include "src/ray/protobuf/gcs.pb.h" +namespace ray { +namespace gcs { + +/// GcsHeartbeatManager is responsible for monitoring nodes liveness as well as +/// handing heartbeat rpc requests. This class is not thread-safe. +class GcsHeartbeatManager : public rpc::HeartbeatInfoHandler { + public: + /// Create a GcsHeartbeatManager. + /// + /// \param io_service The event loop to run the monitor on. + /// \param on_node_death_callback Callback that will be called when node death is + /// detected. + explicit GcsHeartbeatManager( + boost::asio::io_service &io_service, + std::function on_node_death_callback); + + /// Handle heartbeat rpc come from raylet. + void HandleReportHeartbeat(const rpc::ReportHeartbeatRequest &request, + rpc::ReportHeartbeatReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + + /// Start node failure detect loop. + void Start(); + + /// Stop node failure detect loop. + void Stop(); + + /// Register node to this detector. + /// Only if the node has registered, its heartbeat data will be accepted. + /// + /// \param node_id ID of the node to be registered. + void AddNode(const NodeID &node_id); + + protected: + /// A periodic timer that fires on every heartbeat period. Raylets that have + /// not sent a heartbeat within the last num_heartbeats_timeout ticks will be + /// marked as dead in the client table. + void Tick(); + + /// Check that if any raylet is inactive due to no heartbeat for a period of time. + /// If found any, mark it as dead. + void DetectDeadNodes(); + + /// Schedule another tick after a short time. + void ScheduleTick(); + + private: + /// The main event loop for node failure detector. + boost::asio::io_service &io_service_; + std::unique_ptr io_service_thread_; + /// The callback of node death. + std::function on_node_death_callback_; + /// The number of heartbeats that can be missed before a node is removed. + int64_t num_heartbeats_timeout_; + /// A timer that ticks every heartbeat_timeout_ms_ milliseconds. + boost::asio::deadline_timer detect_timer_; + /// For each Raylet that we receive a heartbeat from, the number of ticks + /// that may pass before the Raylet will be declared dead. + absl::flat_hash_map heartbeats_; + /// Is the detect started. + bool is_started_ = false; +}; + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index 3860daee2..820d3a723 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -22,113 +22,18 @@ namespace ray { namespace gcs { -GcsNodeManager::NodeFailureDetector::NodeFailureDetector( - boost::asio::io_service &io_service, - std::shared_ptr gcs_table_storage, - std::shared_ptr gcs_pub_sub, - std::function on_node_death_callback) - : gcs_table_storage_(std::move(gcs_table_storage)), - on_node_death_callback_(std::move(on_node_death_callback)), - num_heartbeats_timeout_(RayConfig::instance().num_heartbeats_timeout()), - light_heartbeat_enabled_(RayConfig::instance().light_heartbeat_enabled()), - detect_timer_(io_service), - gcs_pub_sub_(std::move(gcs_pub_sub)) {} - -void GcsNodeManager::NodeFailureDetector::Start() { - if (!is_started_) { - Tick(); - is_started_ = true; - } -} - -void GcsNodeManager::NodeFailureDetector::AddNode(const NodeID &node_id) { - heartbeats_.emplace(node_id, num_heartbeats_timeout_); -} - -void GcsNodeManager::NodeFailureDetector::HandleHeartbeat(const NodeID &node_id) { - auto iter = heartbeats_.find(node_id); - if (iter == heartbeats_.end()) { - // Ignore this heartbeat as the node is not registered. - // TODO(Shanly): Maybe we should reply the raylet with an error. So the raylet can - // crash itself as soon as possible. - return; - } - - iter->second = num_heartbeats_timeout_; -} - -/// A periodic timer that checks for timed out clients. -void GcsNodeManager::NodeFailureDetector::Tick() { - DetectDeadNodes(); - ScheduleTick(); -} - -void GcsNodeManager::NodeFailureDetector::DetectDeadNodes() { - for (auto it = heartbeats_.begin(); it != heartbeats_.end();) { - auto current = it++; - current->second = current->second - 1; - if (current->second == 0) { - auto node_id = current->first; - RAY_LOG(WARNING) << "Node timed out: " << node_id; - heartbeats_.erase(current); - if (on_node_death_callback_) { - on_node_death_callback_(node_id); - } - } - } -} - -void GcsNodeManager::NodeFailureDetector::ScheduleTick() { - auto heartbeat_period = boost::posix_time::milliseconds( - RayConfig::instance().raylet_heartbeat_timeout_milliseconds()); - detect_timer_.expires_from_now(heartbeat_period); - detect_timer_.async_wait([this](const boost::system::error_code &error) { - if (error == boost::asio::error::operation_aborted) { - // `operation_aborted` is set when `detect_timer_` is canceled or destroyed. - // The Monitor lifetime may be short than the object who use it. (e.g. gcs_server) - return; - } - RAY_CHECK(!error) << "Checking heartbeat failed with error: " << error.message(); - Tick(); - }); -} - ////////////////////////////////////////////////////////////////////////////////////////// GcsNodeManager::GcsNodeManager( - boost::asio::io_service &main_io_service, - boost::asio::io_service &node_failure_detector_io_service, - std::shared_ptr gcs_pub_sub, + boost::asio::io_service &main_io_service, std::shared_ptr gcs_pub_sub, std::shared_ptr gcs_table_storage, std::shared_ptr gcs_resource_manager) - : main_io_service_(main_io_service), - node_failure_detector_(new NodeFailureDetector( - node_failure_detector_io_service, gcs_table_storage, gcs_pub_sub, - [this](const NodeID &node_id) { - // Post this to main event loop to avoid potential concurrency issues. - main_io_service_.post([this, node_id] { - if (auto node = RemoveNode(node_id, /* is_intended = */ false)) { - node->set_state(rpc::GcsNodeInfo::DEAD); - node->set_timestamp(current_sys_time_ms()); - AddDeadNodeToCache(node); - auto on_done = [this, node_id, node](const Status &status) { - auto on_done = [this, node_id, node](const Status &status) { - RAY_CHECK_OK(gcs_pub_sub_->Publish( - NODE_CHANNEL, node_id.Hex(), node->SerializeAsString(), nullptr)); - }; - RAY_CHECK_OK( - gcs_table_storage_->NodeResourceTable().Delete(node_id, on_done)); - }; - RAY_CHECK_OK( - gcs_table_storage_->NodeTable().Put(node_id, *node, on_done)); - } - }); - })), - node_failure_detector_service_(node_failure_detector_io_service), - heartbeat_timer_(main_io_service), + : resource_timer_(main_io_service), + light_report_resource_usage_enabled_( + RayConfig::instance().light_report_resource_usage_enabled()), gcs_pub_sub_(gcs_pub_sub), gcs_table_storage_(gcs_table_storage), gcs_resource_manager_(gcs_resource_manager) { - SendBatchedHeartbeat(); + SendBatchedResourceUsage(); } void GcsNodeManager::HandleRegisterNode(const rpc::RegisterNodeRequest &request, @@ -192,132 +97,27 @@ void GcsNodeManager::HandleGetAllNodeInfo(const rpc::GetAllNodeInfoRequest &requ ++counts_[CountType::GET_ALL_NODE_INFO_REQUEST]; } -void GcsNodeManager::HandleReportHeartbeat(const rpc::ReportHeartbeatRequest &request, - rpc::ReportHeartbeatReply *reply, - rpc::SendReplyCallback send_reply_callback) { - NodeID node_id = NodeID::FromBinary(request.heartbeat().node_id()); - auto heartbeat_data = std::make_shared(); - heartbeat_data->CopyFrom(request.heartbeat()); - - UpdateNodeHeartbeat(node_id, request); - - // Update node realtime resources. - UpdateNodeRealtimeResources(node_id, *heartbeat_data); - - if (!RayConfig::instance().light_heartbeat_enabled() || - heartbeat_data->should_global_gc() || heartbeat_data->resources_total_size() > 0 || - heartbeat_data->resources_available_changed() || - heartbeat_data->resource_load_changed()) { - heartbeat_buffer_[node_id] = *heartbeat_data; - } - - // Note: To avoid heartbeats being delayed by main thread, make sure heartbeat is always - // handled by its own IO service. - node_failure_detector_service_.post( - [this, node_id] { node_failure_detector_->HandleHeartbeat(node_id); }); - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); - ++counts_[CountType::REPORT_HEARTBEAT_REQUEST]; -} - -// TODO(WangTao): Implenent this to handle resource usage report. Basically move resources -// related operations in `HandleReportHeartbeat`. void GcsNodeManager::HandleReportResourceUsage( const rpc::ReportResourceUsageRequest &request, rpc::ReportResourceUsageReply *reply, rpc::SendReplyCallback send_reply_callback) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); -} + NodeID node_id = NodeID::FromBinary(request.resources().node_id()); + auto resources_data = std::make_shared(); + resources_data->CopyFrom(request.resources()); -// TODO(WangTao): Implement this. Basically copy from `HandleGetAllHeartbeat`. -void GcsNodeManager::HandleGetAllResourceUsage( - const rpc::GetAllResourceUsageRequest &request, rpc::GetAllResourceUsageReply *reply, - rpc::SendReplyCallback send_reply_callback) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); -} + UpdateNodeResourceUsage(node_id, request); -void GcsNodeManager::HandleGetResources(const rpc::GetResourcesRequest &request, - rpc::GetResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) { - NodeID node_id = NodeID::FromBinary(request.node_id()); - auto iter = cluster_resources_.find(node_id); - if (iter != cluster_resources_.end()) { - for (auto &resource : iter->second.items()) { - (*reply->mutable_resources())[resource.first] = resource.second; - } + // Update node realtime resources. + UpdateNodeRealtimeResources(node_id, *resources_data); + + if (!light_report_resource_usage_enabled_ || resources_data->should_global_gc() || + resources_data->resources_total_size() > 0 || + resources_data->resources_available_changed() || + resources_data->resource_load_changed()) { + resources_buffer_[node_id] = *resources_data; } + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); - ++counts_[CountType::GET_RESOURCES_REQUEST]; -} - -void GcsNodeManager::HandleUpdateResources(const rpc::UpdateResourcesRequest &request, - rpc::UpdateResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) { - NodeID node_id = NodeID::FromBinary(request.node_id()); - RAY_LOG(DEBUG) << "Updating resources, node id = " << node_id; - auto iter = cluster_resources_.find(node_id); - auto to_be_updated_resources = request.resources(); - if (iter != cluster_resources_.end()) { - for (auto &entry : to_be_updated_resources) { - (*iter->second.mutable_items())[entry.first] = entry.second; - } - auto on_done = [this, node_id, to_be_updated_resources, reply, - send_reply_callback](const Status &status) { - RAY_CHECK_OK(status); - rpc::NodeResourceChange node_resource_change; - node_resource_change.set_node_id(node_id.Binary()); - for (auto &it : to_be_updated_resources) { - (*node_resource_change.mutable_updated_resources())[it.first] = - it.second.resource_capacity(); - } - RAY_CHECK_OK(gcs_pub_sub_->Publish(NODE_RESOURCE_CHANNEL, node_id.Hex(), - node_resource_change.SerializeAsString(), - nullptr)); - - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - RAY_LOG(DEBUG) << "Finished updating resources, node id = " << node_id; - }; - - RAY_CHECK_OK( - gcs_table_storage_->NodeResourceTable().Put(node_id, iter->second, on_done)); - } else { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::Invalid("Node is not exist.")); - RAY_LOG(ERROR) << "Failed to update resources as node " << node_id - << " is not registered."; - } - ++counts_[CountType::UPDATE_RESOURCES_REQUEST]; -} - -void GcsNodeManager::HandleDeleteResources(const rpc::DeleteResourcesRequest &request, - rpc::DeleteResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) { - NodeID node_id = NodeID::FromBinary(request.node_id()); - RAY_LOG(DEBUG) << "Deleting node resources, node id = " << node_id; - auto resource_names = VectorFromProtobuf(request.resource_name_list()); - auto iter = cluster_resources_.find(node_id); - if (iter != cluster_resources_.end()) { - for (auto &resource_name : resource_names) { - RAY_IGNORE_EXPR(iter->second.mutable_items()->erase(resource_name)); - } - auto on_done = [this, node_id, resource_names, reply, - send_reply_callback](const Status &status) { - RAY_CHECK_OK(status); - rpc::NodeResourceChange node_resource_change; - node_resource_change.set_node_id(node_id.Binary()); - for (const auto &resource_name : resource_names) { - node_resource_change.add_deleted_resources(resource_name); - } - RAY_CHECK_OK(gcs_pub_sub_->Publish(NODE_RESOURCE_CHANNEL, node_id.Hex(), - node_resource_change.SerializeAsString(), - nullptr)); - - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - }; - RAY_CHECK_OK( - gcs_table_storage_->NodeResourceTable().Put(node_id, iter->second, on_done)); - } else { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); - RAY_LOG(DEBUG) << "Finished deleting node resources, node id = " << node_id; - } - ++counts_[CountType::DELETE_RESOURCES_REQUEST]; + ++counts_[CountType::REPORT_RESOURCE_USAGE_REQUEST]; } void GcsNodeManager::HandleSetInternalConfig(const rpc::SetInternalConfigRequest &request, @@ -348,31 +148,15 @@ void GcsNodeManager::HandleGetInternalConfig(const rpc::GetInternalConfigRequest ++counts_[CountType::GET_INTERNAL_CONFIG_REQUEST]; } -void GcsNodeManager::HandleGetAllAvailableResources( - const rpc::GetAllAvailableResourcesRequest &request, - rpc::GetAllAvailableResourcesReply *reply, +void GcsNodeManager::HandleGetAllResourceUsage( + const rpc::GetAllResourceUsageRequest &request, rpc::GetAllResourceUsageReply *reply, rpc::SendReplyCallback send_reply_callback) { - for (const auto &iter : gcs_resource_manager_->GetClusterResources()) { - rpc::AvailableResources resource; - resource.set_node_id(iter.first.Binary()); - for (const auto &res : iter.second.GetResourceAmountMap()) { - (*resource.mutable_resources_available())[res.first] = res.second.ToDouble(); - } - reply->add_resources_list()->CopyFrom(resource); - } - GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); - ++counts_[CountType::GET_ALL_AVAILABLE_RESOURCES_REQUEST]; -} - -void GcsNodeManager::HandleGetAllHeartbeat(const rpc::GetAllHeartbeatRequest &request, - rpc::GetAllHeartbeatReply *reply, - rpc::SendReplyCallback send_reply_callback) { - if (!node_heartbeats_.empty()) { - auto batch = std::make_shared(); + if (!node_resource_usages_.empty()) { + auto batch = std::make_shared(); absl::flat_hash_map aggregate_load; - for (const auto &heartbeat : node_heartbeats_) { + for (const auto &usage : node_resource_usages_) { // Aggregate the load reported by each raylet. - auto load = heartbeat.second.resource_load_by_shape(); + auto load = usage.second.resource_load_by_shape(); for (const auto &demand : load.resource_demands()) { auto scheduling_key = ResourceSet(MapFromProtobuf(demand.shape())); auto &aggregate_demand = aggregate_load[scheduling_key]; @@ -388,7 +172,7 @@ void GcsNodeManager::HandleGetAllHeartbeat(const rpc::GetAllHeartbeatRequest &re } } - batch->add_batch()->CopyFrom(heartbeat.second); + batch->add_batch()->CopyFrom(usage.second); } for (const auto &demand : aggregate_load) { @@ -406,38 +190,37 @@ void GcsNodeManager::HandleGetAllHeartbeat(const rpc::GetAllHeartbeatRequest &re auto placement_group_load_proto = batch->mutable_placement_group_load(); placement_group_load_proto->CopyFrom(*placement_group_load.get()); } - reply->mutable_heartbeat_data()->CopyFrom(*batch); + reply->mutable_resource_usage_data()->CopyFrom(*batch); } GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); - ++counts_[CountType::GET_ALL_HEARTBEAT_REQUEST]; + ++counts_[CountType::GET_ALL_RESOURCE_USAGE_REQUEST]; } -void GcsNodeManager::UpdateNodeHeartbeat(const NodeID node_id, - const rpc::ReportHeartbeatRequest &request) { - auto iter = node_heartbeats_.find(node_id); - if (!RayConfig::instance().light_heartbeat_enabled() || - iter == node_heartbeats_.end()) { - auto heartbeat_data = std::make_shared(); - heartbeat_data->CopyFrom(request.heartbeat()); - node_heartbeats_[node_id] = *heartbeat_data; +void GcsNodeManager::UpdateNodeResourceUsage( + const NodeID node_id, const rpc::ReportResourceUsageRequest &request) { + auto iter = node_resource_usages_.find(node_id); + if (!light_report_resource_usage_enabled_ || iter == node_resource_usages_.end()) { + auto resources_data = std::make_shared(); + resources_data->CopyFrom(request.resources()); + node_resource_usages_[node_id] = *resources_data; } else { - if (request.heartbeat().resources_total_size() > 0) { - (*iter->second.mutable_resources_total()) = request.heartbeat().resources_total(); + if (request.resources().resources_total_size() > 0) { + (*iter->second.mutable_resources_total()) = request.resources().resources_total(); } - if (request.heartbeat().resources_available_changed()) { + if (request.resources().resources_available_changed()) { (*iter->second.mutable_resources_available()) = - request.heartbeat().resources_available(); + request.resources().resources_available(); } - if (request.heartbeat().resource_load_changed()) { - (*iter->second.mutable_resource_load()) = request.heartbeat().resource_load(); + if (request.resources().resource_load_changed()) { + (*iter->second.mutable_resource_load()) = request.resources().resource_load(); } (*iter->second.mutable_resource_load_by_shape()) = - request.heartbeat().resource_load_by_shape(); + request.resources().resource_load_by_shape(); } } -absl::optional> GcsNodeManager::GetNode( +absl::optional> GcsNodeManager::GetAliveNode( const ray::NodeID &node_id) const { auto iter = alive_nodes_.find(node_id); if (iter == alive_nodes_.end()) { @@ -452,18 +235,12 @@ void GcsNodeManager::AddNode(std::shared_ptr node) { auto iter = alive_nodes_.find(node_id); if (iter == alive_nodes_.end()) { alive_nodes_.emplace(node_id, node); - // Add an empty resources for this node. - RAY_CHECK(cluster_resources_.emplace(node_id, rpc::ResourceMap()).second); - // Register this node to the `node_failure_detector_` which will start monitoring it. - // Note: To avoid heartbeats being delayed by main thread, make sure node addition is - // always handled by its own IO service. - node_failure_detector_service_.post( - [this, node_id] { node_failure_detector_->AddNode(node_id); }); // Notify all listeners. for (auto &listener : node_added_listeners_) { listener(node); } + gcs_resource_manager_->OnNodeAdd(node_id); } } @@ -479,8 +256,8 @@ std::shared_ptr GcsNodeManager::RemoveNode( // Remove from alive nodes. alive_nodes_.erase(iter); // Remove from cluster resources. - cluster_resources_.erase(node_id); - heartbeat_buffer_.erase(node_id); + gcs_resource_manager_->OnNodeDead(node_id); + resources_buffer_.erase(node_id); if (!is_intended) { // Broadcast a warning to all of the drivers indicating that the node // has been marked as dead. @@ -505,11 +282,25 @@ std::shared_ptr GcsNodeManager::RemoveNode( return removed_node; } +void GcsNodeManager::OnNodeFailure(const NodeID &node_id) { + if (auto node = RemoveNode(node_id, /* is_intended = */ false)) { + node->set_state(rpc::GcsNodeInfo::DEAD); + node->set_timestamp(current_sys_time_ms()); + AddDeadNodeToCache(node); + auto on_done = [this, node_id, node](const Status &status) { + auto on_done = [this, node_id, node](const Status &status) { + RAY_CHECK_OK(gcs_pub_sub_->Publish(NODE_CHANNEL, node_id.Hex(), + node->SerializeAsString(), nullptr)); + }; + RAY_CHECK_OK(gcs_table_storage_->NodeResourceTable().Delete(node_id, on_done)); + }; + RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put(node_id, *node, on_done)); + } +} + void GcsNodeManager::Initialize(const GcsInitData &gcs_init_data) { for (const auto &item : gcs_init_data.Nodes()) { if (item.second.state() == rpc::GcsNodeInfo::ALIVE) { - // Call `AddNode` for this node to make sure it is tracked by the failure - // detector. AddNode(std::make_shared(item.second)); } else if (item.second.state() == rpc::GcsNodeInfo::DEAD) { dead_nodes_.emplace(item.first, std::make_shared(item.second)); @@ -519,27 +310,15 @@ void GcsNodeManager::Initialize(const GcsInitData &gcs_init_data) { sorted_dead_node_list_.sort( [](const std::pair &left, const std::pair &right) { return left.second < right.second; }); - - for (auto &entry : gcs_init_data.ClusterResources()) { - if (alive_nodes_.count(entry.first)) { - cluster_resources_[entry.first] = entry.second; - } - } -} - -void GcsNodeManager::StartNodeFailureDetector() { - // Note: To avoid heartbeats being delayed by main thread, make sure detector start is - // always handled by its own IO service. - node_failure_detector_service_.post([this] { node_failure_detector_->Start(); }); } void GcsNodeManager::UpdateNodeRealtimeResources( - const NodeID &node_id, const rpc::HeartbeatTableData &heartbeat) { - if (!RayConfig::instance().light_heartbeat_enabled() || + const NodeID &node_id, const rpc::ResourcesData &resource_data) { + if (!light_report_resource_usage_enabled_ || gcs_resource_manager_->GetClusterResources().count(node_id) == 0 || - heartbeat.resources_available_changed()) { - gcs_resource_manager_->UpdateResources( - node_id, ResourceSet(MapFromProtobuf(heartbeat.resources_available()))); + resource_data.resources_available_changed()) { + gcs_resource_manager_->SetAvailableResources( + node_id, ResourceSet(MapFromProtobuf(resource_data.resources_available()))); } } @@ -560,31 +339,31 @@ void GcsNodeManager::AddDeadNodeToCache(std::shared_ptr node) sorted_dead_node_list_.emplace_back(node_id, node->timestamp()); } -void GcsNodeManager::SendBatchedHeartbeat() { - if (!heartbeat_buffer_.empty()) { - auto batch = std::make_shared(); - for (auto &heartbeat : heartbeat_buffer_) { - batch->add_batch()->Swap(&heartbeat.second); +void GcsNodeManager::SendBatchedResourceUsage() { + if (!resources_buffer_.empty()) { + auto batch = std::make_shared(); + for (auto &resources : resources_buffer_) { + batch->add_batch()->Swap(&resources.second); } stats::OutboundHeartbeatSizeKB.Record((double)(batch->ByteSizeLong() / 1024.0)); - RAY_CHECK_OK(gcs_pub_sub_->Publish(HEARTBEAT_BATCH_CHANNEL, "", + RAY_CHECK_OK(gcs_pub_sub_->Publish(RESOURCES_BATCH_CHANNEL, "", batch->SerializeAsString(), nullptr)); - heartbeat_buffer_.clear(); + resources_buffer_.clear(); } - auto heartbeat_period = boost::posix_time::milliseconds( - RayConfig::instance().raylet_heartbeat_timeout_milliseconds()); - heartbeat_timer_.expires_from_now(heartbeat_period); - heartbeat_timer_.async_wait([this](const boost::system::error_code &error) { + auto resources_period = boost::posix_time::milliseconds( + RayConfig::instance().raylet_report_resources_period_milliseconds()); + resource_timer_.expires_from_now(resources_period); + resource_timer_.async_wait([this](const boost::system::error_code &error) { if (error == boost::asio::error::operation_aborted) { - // `operation_aborted` is set when `heartbeat_timer_` is canceled or destroyed. + // `operation_aborted` is set when `resource_timer_` is canceled or destroyed. // The Monitor lifetime may be short than the object who use it. (e.g. gcs_server) return; } - RAY_CHECK(!error) << "Sending batched heartbeat failed with error: " + RAY_CHECK(!error) << "Sending batched resource usage failed with error: " << error.message(); - SendBatchedHeartbeat(); + SendBatchedResourceUsage(); }); } @@ -596,16 +375,10 @@ std::string GcsNodeManager::DebugString() const { << counts_[CountType::UNREGISTER_NODE_REQUEST] << ", GetAllNodeInfo request count: " << counts_[CountType::GET_ALL_NODE_INFO_REQUEST] - << ", ReportHeartbeat request count: " - << counts_[CountType::REPORT_HEARTBEAT_REQUEST] - << ", GetHeartbeat request count: " << counts_[CountType::GET_HEARTBEAT_REQUEST] - << ", GetAllHeartbeat request count: " - << counts_[CountType::GET_ALL_HEARTBEAT_REQUEST] - << ", GetResources request count: " << counts_[CountType::GET_RESOURCES_REQUEST] - << ", UpdateResources request count: " - << counts_[CountType::UPDATE_RESOURCES_REQUEST] - << ", DeleteResources request count: " - << counts_[CountType::DELETE_RESOURCES_REQUEST] + << ", ReportResourceUsage request count: " + << counts_[CountType::REPORT_RESOURCE_USAGE_REQUEST] + << ", GetAllResourceUsage request count: " + << counts_[CountType::GET_ALL_RESOURCE_USAGE_REQUEST] << ", SetInternalConfig request count: " << counts_[CountType::SET_INTERNAL_CONFIG_REQUEST] << ", GetInternalConfig request count: " diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index 41b70b957..3c62af3a7 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -37,12 +37,10 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// Create a GcsNodeManager. /// /// \param main_io_service The main event loop. - /// \param node_failure_detector_io_service The event loop of node failure detector. /// \param gcs_pub_sub GCS message publisher. /// \param gcs_table_storage GCS table external storage accessor. /// \param gcs_resource_manager GCS resource manager. explicit GcsNodeManager(boost::asio::io_service &main_io_service, - boost::asio::io_service &node_failure_detector_io_service, std::shared_ptr gcs_pub_sub, std::shared_ptr gcs_table_storage, std::shared_ptr gcs_resource_manager); @@ -62,11 +60,6 @@ class GcsNodeManager : public rpc::NodeInfoHandler { rpc::GetAllNodeInfoReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle heartbeat rpc come from raylet. - void HandleReportHeartbeat(const rpc::ReportHeartbeatRequest &request, - rpc::ReportHeartbeatReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - /// Handle report resource usage rpc come from raylet. void HandleReportResourceUsage(const rpc::ReportResourceUsageRequest &request, rpc::ReportResourceUsageReply *reply, @@ -77,21 +70,6 @@ class GcsNodeManager : public rpc::NodeInfoHandler { rpc::GetAllResourceUsageReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle get resource rpc request. - void HandleGetResources(const rpc::GetResourcesRequest &request, - rpc::GetResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - - /// Handle update resource rpc request. - void HandleUpdateResources(const rpc::UpdateResourcesRequest &request, - rpc::UpdateResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - - /// Handle delete resource rpc request. - void HandleDeleteResources(const rpc::DeleteResourcesRequest &request, - rpc::DeleteResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - /// Handle set internal config. void HandleSetInternalConfig(const rpc::SetInternalConfigRequest &request, rpc::SetInternalConfigReply *reply, @@ -102,23 +80,14 @@ class GcsNodeManager : public rpc::NodeInfoHandler { rpc::GetInternalConfigReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle get available resources of all nodes. - void HandleGetAllAvailableResources( - const rpc::GetAllAvailableResourcesRequest &request, - rpc::GetAllAvailableResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - - /// Handle get all heartbeat rpc request. Only used when light heartbeat enabled. - void HandleGetAllHeartbeat(const rpc::GetAllHeartbeatRequest &request, - rpc::GetAllHeartbeatReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - - /// Update heartbeat of given node. + /// Update resource usage of given node. /// /// \param node_id Node id. - /// \param request Request containing heartbeat. - void UpdateNodeHeartbeat(const NodeID node_id, - const rpc::ReportHeartbeatRequest &request); + /// \param request Request containing resource usage. + void UpdateNodeResourceUsage(const NodeID node_id, + const rpc::ReportResourceUsageRequest &request); + + void OnNodeFailure(const NodeID &node_id); /// Add an alive node. /// @@ -137,7 +106,8 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// /// \param node_id The id of the node. /// \return the node if it is alive. Optional empty value if it is not alive. - absl::optional> GetNode(const NodeID &node_id) const; + absl::optional> GetAliveNode( + const NodeID &node_id) const; /// Get all alive nodes. /// @@ -171,12 +141,9 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// \param gcs_init_data. void Initialize(const GcsInitData &gcs_init_data); - /// Start node failure detector. - void StartNodeFailureDetector(); - // Update node realtime resources. void UpdateNodeRealtimeResources(const NodeID &node_id, - const rpc::HeartbeatTableData &heartbeat); + const rpc::ResourcesData &heartbeat); /// Update the placement group load information so that it will be reported through /// heartbeat. @@ -187,72 +154,6 @@ class GcsNodeManager : public rpc::NodeInfoHandler { std::string DebugString() const; - protected: - class NodeFailureDetector { - public: - /// Create a NodeFailureDetector. - /// - /// \param io_service The event loop to run the monitor on. - /// \param gcs_table_storage GCS table external storage accessor. - /// \param gcs_pub_sub GCS message publisher. - /// \param on_node_death_callback Callback that will be called when node death is - /// detected. - explicit NodeFailureDetector( - boost::asio::io_service &io_service, - std::shared_ptr gcs_table_storage, - std::shared_ptr gcs_pub_sub, - std::function on_node_death_callback); - - // Note: To avoid heartbeats being delayed by main thread, all public methods below - // should be posted to its own IO service. - - /// Start failure detector. - void Start(); - - /// Register node to this detector. - /// Only if the node has registered, its heartbeat data will be accepted. - /// - /// \param node_id ID of the node to be registered. - void AddNode(const NodeID &node_id); - - /// Handle a heartbeat from a Raylet. - /// - /// \param node_id The node ID of the Raylet that sent the heartbeat. - void HandleHeartbeat(const NodeID &node_id); - - protected: - /// A periodic timer that fires on every heartbeat period. Raylets that have - /// not sent a heartbeat within the last num_heartbeats_timeout ticks will be - /// marked as dead in the client table. - void Tick(); - - /// Check that if any raylet is inactive due to no heartbeat for a period of time. - /// If found any, mark it as dead. - void DetectDeadNodes(); - - /// Schedule another tick after a short time. - void ScheduleTick(); - - protected: - /// Storage for GCS tables. - std::shared_ptr gcs_table_storage_; - /// The callback of node death. - std::function on_node_death_callback_; - /// The number of heartbeats that can be missed before a node is removed. - int64_t num_heartbeats_timeout_; - // Only the changed part will be included in heartbeat if this is true. - const bool light_heartbeat_enabled_; - /// A timer that ticks every heartbeat_timeout_ms_ milliseconds. - boost::asio::deadline_timer detect_timer_; - /// For each Raylet that we receive a heartbeat from, the number of ticks - /// that may pass before the Raylet will be declared dead. - absl::flat_hash_map heartbeats_; - /// A publisher for publishing gcs messages. - std::shared_ptr gcs_pub_sub_; - /// Is the detect started. - bool is_started_ = false; - }; - private: /// Add the dead node to the cache. If the cache is full, the earliest dead node is /// evicted. @@ -260,17 +161,13 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// \param node The node which is dead. void AddDeadNodeToCache(std::shared_ptr node); - /// Send any buffered heartbeats as a single publish. - void SendBatchedHeartbeat(); + /// Send any buffered resource usage as a single publish. + void SendBatchedResourceUsage(); - /// The main event loop for node failure detector. - boost::asio::io_service &main_io_service_; - /// Detector to detect the failure of node. - std::unique_ptr node_failure_detector_; - /// The event loop for node failure detector. - boost::asio::io_service &node_failure_detector_service_; - /// A timer that ticks every heartbeat_timeout_ms_ milliseconds. - boost::asio::deadline_timer heartbeat_timer_; + /// A timer that ticks every raylet_report_resources_period_milliseconds. + boost::asio::deadline_timer resource_timer_; + // Only the changed part will be reported if this is true. + const bool light_report_resource_usage_enabled_; /// Alive nodes. absl::flat_hash_map> alive_nodes_; /// Dead nodes. @@ -278,12 +175,10 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// The nodes are sorted according to the timestamp, and the oldest is at the head of /// the list. std::list> sorted_dead_node_list_; - /// Cluster resources. - absl::flat_hash_map cluster_resources_; - /// Newest heartbeat of all nodes. - absl::flat_hash_map node_heartbeats_; - /// A buffer containing heartbeats received from node managers in the last tick. - absl::flat_hash_map heartbeat_buffer_; + /// Newest resource usage of all nodes. + absl::flat_hash_map node_resource_usages_; + /// A buffer containing resource usage received from node managers in the last tick. + absl::flat_hash_map resources_buffer_; /// Listeners which monitors the addition of nodes. std::vector)>> node_added_listeners_; @@ -304,16 +199,11 @@ class GcsNodeManager : public rpc::NodeInfoHandler { REGISTER_NODE_REQUEST = 0, UNREGISTER_NODE_REQUEST = 1, GET_ALL_NODE_INFO_REQUEST = 2, - REPORT_HEARTBEAT_REQUEST = 3, - GET_HEARTBEAT_REQUEST = 4, - GET_ALL_HEARTBEAT_REQUEST = 5, - GET_RESOURCES_REQUEST = 6, - UPDATE_RESOURCES_REQUEST = 7, - DELETE_RESOURCES_REQUEST = 8, - SET_INTERNAL_CONFIG_REQUEST = 9, - GET_INTERNAL_CONFIG_REQUEST = 10, - GET_ALL_AVAILABLE_RESOURCES_REQUEST = 11, - CountType_MAX = 12, + REPORT_RESOURCE_USAGE_REQUEST = 3, + GET_ALL_RESOURCE_USAGE_REQUEST = 4, + SET_INTERNAL_CONFIG_REQUEST = 5, + GET_INTERNAL_CONFIG_REQUEST = 6, + CountType_MAX = 7, }; uint64_t counts_[CountType::CountType_MAX] = {0}; }; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc index d6babd48e..5ac3d0510 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc @@ -227,6 +227,18 @@ void GcsPlacementGroupManager::OnPlacementGroupCreationSuccess( } MarkSchedulingDone(); SchedulePendingPlacementGroups(); + + // Invoke all callbacks for all `WaitPlacementGroupUntilReady` requests of this + // placement group and remove all of them from + // placement_group_to_create_callbacks_. + auto pg_to_create_iter = + placement_group_to_create_callbacks_.find(placement_group_id); + if (pg_to_create_iter != placement_group_to_create_callbacks_.end()) { + for (auto &callback : pg_to_create_iter->second) { + callback(status); + } + placement_group_to_create_callbacks_.erase(pg_to_create_iter); + } })); } @@ -301,6 +313,7 @@ void GcsPlacementGroupManager::RemovePlacementGroup( } auto placement_group = placement_group_it->second; registered_placement_groups_.erase(placement_group_it); + placement_group_to_create_callbacks_.erase(placement_group_id); // Destroy all bundles. gcs_placement_group_scheduler_->DestroyPlacementGroupBundleResourcesIfExists( @@ -388,6 +401,64 @@ void GcsPlacementGroupManager::HandleGetAllPlacementGroup( ++counts_[CountType::GET_ALL_PLACEMENT_GROUP_REQUEST]; } +void GcsPlacementGroupManager::HandleWaitPlacementGroupUntilReady( + const rpc::WaitPlacementGroupUntilReadyRequest &request, + rpc::WaitPlacementGroupUntilReadyReply *reply, + rpc::SendReplyCallback send_reply_callback) { + PlacementGroupID placement_group_id = + PlacementGroupID::FromBinary(request.placement_group_id()); + RAY_LOG(DEBUG) << "Waiting for placement group until ready, placement group id = " + << placement_group_id; + + auto callback = [placement_group_id, reply, send_reply_callback](const Status &status) { + RAY_LOG(DEBUG) + << "Finished waiting for placement group until ready, placement group id = " + << placement_group_id; + GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); + }; + + // If the placement group does not exist or it has been successfully created, return + // directly. + const auto &iter = registered_placement_groups_.find(placement_group_id); + if (iter == registered_placement_groups_.end()) { + // Check whether the placement group does not exist or is removed. + auto on_done = [this, placement_group_id, reply, callback, send_reply_callback]( + const Status &status, + const boost::optional &result) { + if (result) { + RAY_LOG(DEBUG) << "Placement group is removed, placement group id = " + << placement_group_id; + GCS_RPC_SEND_REPLY(send_reply_callback, reply, + Status::NotFound("Placement group is removed.")); + } else { + // `wait` is a method of placement group object. Placement group object is + // obtained by create placement group api, so it can guarantee the existence of + // placement group. + // GCS client does not guarantee the order of placement group creation and + // wait, so GCS may call wait placement group first and then create placement + // group. + placement_group_to_create_callbacks_[placement_group_id].emplace_back( + std::move(callback)); + } + }; + + Status status = + gcs_table_storage_->PlacementGroupTable().Get(placement_group_id, on_done); + if (!status.ok()) { + on_done(status, boost::none); + } + } else if (iter->second->GetState() == rpc::PlacementGroupTableData::CREATED) { + RAY_LOG(DEBUG) << "Placement group is created, placement group id = " + << placement_group_id; + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); + } else { + placement_group_to_create_callbacks_[placement_group_id].emplace_back( + std::move(callback)); + } + + ++counts_[CountType::WAIT_PLACEMENT_GROUP_UNTIL_READY_REQUEST]; +} + void GcsPlacementGroupManager::RetryCreatingPlacementGroup() { execute_after(io_context_, [this] { SchedulePendingPlacementGroups(); }, RayConfig::instance().gcs_create_placement_group_retry_interval_ms()); @@ -509,6 +580,8 @@ std::string GcsPlacementGroupManager::DebugString() const { << counts_[CountType::GET_PLACEMENT_GROUP_REQUEST] << ", GetAllPlacementGroup request count: " << counts_[CountType::GET_ALL_PLACEMENT_GROUP_REQUEST] + << ", WaitPlacementGroupUntilReady request count: " + << counts_[CountType::WAIT_PLACEMENT_GROUP_UNTIL_READY_REQUEST] << ", Registered placement groups count: " << registered_placement_groups_.size() << ", Pending placement groups count: " << pending_placement_groups_.size() << "}"; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h index eec2048d6..17a4f5c11 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h @@ -155,6 +155,11 @@ class GcsPlacementGroupManager : public rpc::PlacementGroupInfoHandler { rpc::GetAllPlacementGroupReply *reply, rpc::SendReplyCallback send_reply_callback) override; + void HandleWaitPlacementGroupUntilReady( + const rpc::WaitPlacementGroupUntilReadyRequest &request, + rpc::WaitPlacementGroupUntilReadyReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + /// Register placement_group asynchronously. /// /// \param placement_group The placement group to be created. @@ -276,6 +281,10 @@ class GcsPlacementGroupManager : public rpc::PlacementGroupInfoHandler { absl::flat_hash_map placement_group_to_register_callback_; + /// Callback of `WaitPlacementGroupUntilReady` requests. + absl::flat_hash_map> + placement_group_to_create_callbacks_; + /// All registered placement_groups (pending placement_groups are also included). absl::flat_hash_map> registered_placement_groups_; @@ -308,7 +317,8 @@ class GcsPlacementGroupManager : public rpc::PlacementGroupInfoHandler { REMOVE_PLACEMENT_GROUP_REQUEST = 1, GET_PLACEMENT_GROUP_REQUEST = 2, GET_ALL_PLACEMENT_GROUP_REQUEST = 3, - CountType_MAX = 4, + WAIT_PLACEMENT_GROUP_UNTIL_READY_REQUEST = 4, + CountType_MAX = 5, }; uint64_t counts_[CountType::CountType_MAX] = {0}; }; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc index 3c6b1ead2..13a7f38e6 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc @@ -24,12 +24,12 @@ GcsPlacementGroupScheduler::GcsPlacementGroupScheduler( boost::asio::io_context &io_context, std::shared_ptr gcs_table_storage, const gcs::GcsNodeManager &gcs_node_manager, GcsResourceManager &gcs_resource_manager, - ReserveResourceClientFactoryFn lease_client_factory) + std::shared_ptr raylet_client_pool) : return_timer_(io_context), gcs_table_storage_(std::move(gcs_table_storage)), gcs_node_manager_(gcs_node_manager), gcs_resource_manager_(gcs_resource_manager), - lease_client_factory_(std::move(lease_client_factory)) { + raylet_client_pool_(raylet_client_pool) { scheduler_strategies_.push_back(std::make_shared()); scheduler_strategies_.push_back(std::make_shared()); scheduler_strategies_.push_back(std::make_shared()); @@ -66,7 +66,7 @@ ScheduleMap GcsStrictPackStrategy::Schedule( // Filter candidate nodes. std::vector> candidate_nodes; for (auto &node : context->cluster_resources_) { - if (required_resources.IsSubset(node.second)) { + if (required_resources.IsSubset(node.second.GetAvailableResources())) { candidate_nodes.emplace_back((*context->node_to_bundles_)[node.first], node.first); } } @@ -99,7 +99,8 @@ ScheduleMap GcsPackStrategy::Schedule( for (const auto &bundle : bundles) { const auto &required_resources = bundle->GetRequiredResources(); for (const auto &node : context->cluster_resources_) { - if (IsAvailableResourceSufficient(node.second, allocated_resources[node.first], + if (IsAvailableResourceSufficient(node.second.GetAvailableResources(), + allocated_resources[node.first], required_resources)) { schedule_map[bundle->BundleId()] = node.first; allocated_resources[node.first].AddResources(required_resources); @@ -135,7 +136,8 @@ ScheduleMap GcsSpreadStrategy::Schedule( // that meets the resource requirements. `iter_begin` is the next node of the last // selected node. for (; iter != candidate_nodes.end(); ++iter) { - if (IsAvailableResourceSufficient(iter->second, allocated_resources[iter->first], + if (IsAvailableResourceSufficient(iter->second.GetAvailableResources(), + allocated_resources[iter->first], required_resources)) { schedule_map[bundle->BundleId()] = iter->first; allocated_resources[iter->first].AddResources(required_resources); @@ -153,8 +155,9 @@ ScheduleMap GcsSpreadStrategy::Schedule( if (iter_begin != candidate_nodes.begin()) { // Traverse all the nodes from `candidate_nodes.begin()` to `iter_begin`. for (iter = candidate_nodes.begin(); iter != iter_begin; ++iter) { - if (IsAvailableResourceSufficient( - iter->second, allocated_resources[iter->first], required_resources)) { + if (IsAvailableResourceSufficient(iter->second.GetAvailableResources(), + allocated_resources[iter->first], + required_resources)) { schedule_map[bundle->BundleId()] = iter->first; allocated_resources[iter->first].AddResources(required_resources); break; @@ -193,13 +196,22 @@ ScheduleMap GcsStrictSpreadStrategy::Schedule( return schedule_map; } + // Filter out the nodes already scheduled by this placement group. absl::flat_hash_map allocated_resources; + if (context->bundle_locations_.has_value()) { + const auto &bundle_locations = context->bundle_locations_.value(); + for (auto &bundle : *bundle_locations) { + allocated_resources[bundle.second.first] = ResourceSet(); + } + } + for (const auto &bundle : bundles) { const auto &required_resources = bundle->GetRequiredResources(); auto iter = candidate_nodes.begin(); for (; iter != candidate_nodes.end(); ++iter) { if (!allocated_resources.contains(iter->first) && - IsAvailableResourceSufficient(iter->second, allocated_resources[iter->first], + IsAvailableResourceSufficient(iter->second.GetAvailableResources(), + allocated_resources[iter->first], required_resources)) { schedule_map[bundle->BundleId()] = iter->first; allocated_resources[iter->first].AddResources(required_resources); @@ -253,7 +265,7 @@ void GcsPlacementGroupScheduler::ScheduleUnplacedBundles( } auto lease_status_tracker = - std::make_shared(placement_group, bundles); + std::make_shared(placement_group, bundles, selected_nodes); RAY_CHECK(placement_group_leasing_in_progress_ .emplace(placement_group->GetPlacementGroupID(), lease_status_tracker) .second); @@ -265,7 +277,7 @@ void GcsPlacementGroupScheduler::ScheduleUnplacedBundles( lease_status_tracker->MarkPreparePhaseStarted(node_id, bundle); // TODO(sang): The callback might not be called at all if nodes are dead. We should // handle this case properly. - PrepareResources(bundle, gcs_node_manager_.GetNode(node_id), + PrepareResources(bundle, gcs_node_manager_.GetAliveNode(node_id), [this, bundle, node_id, lease_status_tracker, failure_callback, success_callback](const Status &status) { lease_status_tracker->MarkPrepareRequestReturned(node_id, bundle, @@ -280,10 +292,18 @@ void GcsPlacementGroupScheduler::ScheduleUnplacedBundles( void GcsPlacementGroupScheduler::DestroyPlacementGroupBundleResourcesIfExists( const PlacementGroupID &placement_group_id) { - // There could be leasing bundles and committed bundles at the same time if placement - // groups are rescheduling. - DestroyPlacementGroupPreparedBundleResources(placement_group_id); - DestroyPlacementGroupCommittedBundleResources(placement_group_id); + auto &bundle_locations = + committed_bundle_location_index_.GetBundleLocations(placement_group_id); + if (bundle_locations.has_value()) { + // There could be leasing bundles and committed bundles at the same time if placement + // groups are rescheduling, so we need to destroy prepared bundles and committed + // bundles at the same time. + DestroyPlacementGroupPreparedBundleResources(placement_group_id); + DestroyPlacementGroupCommittedBundleResources(placement_group_id); + + // Return destroyed bundles resources to the cluster resource. + ReturnBundleResources(bundle_locations.value()); + } } void GcsPlacementGroupScheduler::MarkScheduleCancelled( @@ -371,13 +391,7 @@ void GcsPlacementGroupScheduler::CancelResourceReserve( std::shared_ptr GcsPlacementGroupScheduler::GetOrConnectLeaseClient(const rpc::Address &raylet_address) { - auto node_id = NodeID::FromBinary(raylet_address.raylet_id()); - auto iter = remote_lease_clients_.find(node_id); - if (iter == remote_lease_clients_.end()) { - auto lease_client = lease_client_factory_(raylet_address); - iter = remote_lease_clients_.emplace(node_id, std::move(lease_client)).first; - } - return iter->second; + return raylet_client_pool_->GetOrConnectByAddress(raylet_address); } std::shared_ptr @@ -401,7 +415,7 @@ void GcsPlacementGroupScheduler::CommitAllBundles( lease_status_tracker->MarkCommitPhaseStarted(); for (const auto &bundle_to_commit : *prepared_bundle_locations) { const auto &node_id = bundle_to_commit.second.first; - const auto &node = gcs_node_manager_.GetNode(node_id); + const auto &node = gcs_node_manager_.GetAliveNode(node_id); const auto &bundle = bundle_to_commit.second.second; auto commit_resources_callback = [this, lease_status_tracker, bundle, node_id, @@ -448,6 +462,7 @@ void GcsPlacementGroupScheduler::OnAllBundlePrepareRequestReturned( auto it = placement_group_leasing_in_progress_.find(placement_group_id); RAY_CHECK(it != placement_group_leasing_in_progress_.end()); placement_group_leasing_in_progress_.erase(it); + ReturnBundleResources(lease_status_tracker->GetBundleLocations()); schedule_failure_handler(placement_group); return; } @@ -496,13 +511,24 @@ void GcsPlacementGroupScheduler::OnAllBundleCommitRequestReturned( committed_bundle_location_index_.AddBundleLocations(placement_group_id, prepared_bundle_locations); - // If the placement group scheduling has been cancelled, destroy them. + // NOTE: If the placement group scheduling has been cancelled, we just need to destroy + // the committed bundles. The reason is that only `RemovePlacementGroup` will mark the + // state of placement group as `CANCELLED` and it will also destroy all prepared and + // committed bundles of the placement group. + // However, it cannot destroy the newly submitted bundles in this scheduling, so we need + // to destroy them separately. if (lease_status_tracker->GetLeasingState() == LeasingState::CANCELLED) { - DestroyPlacementGroupBundleResourcesIfExists(placement_group_id); + DestroyPlacementGroupCommittedBundleResources(placement_group_id); + ReturnBundleResources(lease_status_tracker->GetBundleLocations()); schedule_failure_handler(placement_group); return; } + // Acquire bundle resources from gcs resources manager. + const auto &committed_bundle_locations = + lease_status_tracker->GetCommittedBundleLocations(); + AcquireBundleResources(committed_bundle_locations); + if (!lease_status_tracker->AllCommitRequestsSuccessful()) { // Update the state to be reschedule so that the failure handle will reschedule the // failed bundles. @@ -512,11 +538,11 @@ void GcsPlacementGroupScheduler::OnAllBundleCommitRequestReturned( placement_group->GetMutableBundle(bundle.first.second)->clear_node_id(); } placement_group->UpdateState(rpc::PlacementGroupTableData::RESCHEDULING); + ReturnBundleResources(uncommitted_bundle_locations); schedule_failure_handler(placement_group); - return; + } else { + schedule_success_handler(placement_group); } - - schedule_success_handler(placement_group); } std::unique_ptr GcsPlacementGroupScheduler::GetScheduleContext( @@ -601,7 +627,7 @@ void GcsPlacementGroupScheduler::DestroyPlacementGroupPreparedBundleResources( for (const auto &iter : *(leasing_bundle_locations)) { auto &bundle_spec = iter.second.second; auto &node_id = iter.second.first; - CancelResourceReserve(bundle_spec, gcs_node_manager_.GetNode(node_id)); + CancelResourceReserve(bundle_spec, gcs_node_manager_.GetAliveNode(node_id)); } } } @@ -620,12 +646,30 @@ void GcsPlacementGroupScheduler::DestroyPlacementGroupCommittedBundleResources( for (const auto &iter : *(committed_bundle_locations)) { auto &bundle_spec = iter.second.second; auto &node_id = iter.second.first; - CancelResourceReserve(bundle_spec, gcs_node_manager_.GetNode(node_id)); + CancelResourceReserve(bundle_spec, gcs_node_manager_.GetAliveNode(node_id)); } committed_bundle_location_index_.Erase(placement_group_id); } } +void GcsPlacementGroupScheduler::AcquireBundleResources( + const std::shared_ptr &bundle_locations) { + // Acquire bundle resources from gcs resources manager. + for (auto &bundle : *bundle_locations) { + gcs_resource_manager_.AcquireResources(bundle.second.first, + bundle.second.second->GetRequiredResources()); + } +} + +void GcsPlacementGroupScheduler::ReturnBundleResources( + const std::shared_ptr &bundle_locations) { + // Release bundle resources to gcs resources manager. + for (auto &bundle : *bundle_locations) { + gcs_resource_manager_.ReleaseResources(bundle.second.first, + bundle.second.second->GetRequiredResources()); + } +} + void BundleLocationIndex::AddBundleLocations( const PlacementGroupID &placement_group_id, std::shared_ptr bundle_locations) { @@ -726,10 +770,18 @@ void BundleLocationIndex::AddNodes( LeaseStatusTracker::LeaseStatusTracker( std::shared_ptr placement_group, - std::vector> &unplaced_bundles) + const std::vector> &unplaced_bundles, + const ScheduleMap &schedule_map) : placement_group_(placement_group), bundles_to_schedule_(unplaced_bundles) { preparing_bundle_locations_ = std::make_shared(); uncommitted_bundle_locations_ = std::make_shared(); + committed_bundle_locations_ = std::make_shared(); + bundle_locations_ = std::make_shared(); + for (const auto &bundle : unplaced_bundles) { + const auto &iter = schedule_map.find(bundle->BundleId()); + RAY_CHECK(iter != schedule_map.end()); + (*bundle_locations_)[bundle->BundleId()] = std::make_pair(iter->second, bundle); + } } bool LeaseStatusTracker::MarkPreparePhaseStarted( @@ -780,6 +832,8 @@ void LeaseStatusTracker::MarkCommitRequestReturned( const auto &bundle_id = bundle->BundleId(); if (!status.ok()) { uncommitted_bundle_locations_->emplace(bundle_id, std::make_pair(node_id, bundle)); + } else { + committed_bundle_locations_->emplace(bundle_id, std::make_pair(node_id, bundle)); } } @@ -809,6 +863,15 @@ const std::shared_ptr return uncommitted_bundle_locations_; } +const std::shared_ptr &LeaseStatusTracker::GetCommittedBundleLocations() + const { + return committed_bundle_locations_; +} + +const std::shared_ptr &LeaseStatusTracker::GetBundleLocations() const { + return bundle_locations_; +} + const std::vector> &LeaseStatusTracker::GetBundlesToSchedule() const { return bundles_to_schedule_; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index a57441f89..a604513a7 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -22,6 +22,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/node_manager_client.h" +#include "ray/rpc/node_manager/node_manager_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "src/ray/protobuf/gcs_service.pb.h" @@ -64,6 +65,8 @@ class GcsPlacementGroupSchedulerInterface { const NodeID &node_id) = 0; /// Destroy bundle resources from all nodes in the placement group. + /// + /// \param placement_group_id The id of the placement group to be destroyed. virtual void DestroyPlacementGroupBundleResourcesIfExists( const PlacementGroupID &placement_group_id) = 0; @@ -86,9 +89,10 @@ class GcsPlacementGroupSchedulerInterface { /// ScheduleContext provides information that are needed for bundle scheduling decision. class ScheduleContext { public: - ScheduleContext(std::shared_ptr> node_to_bundles, - const absl::optional> bundle_locations, - const absl::flat_hash_map &cluster_resources) + ScheduleContext( + std::shared_ptr> node_to_bundles, + const absl::optional> bundle_locations, + const absl::flat_hash_map &cluster_resources) : node_to_bundles_(std::move(node_to_bundles)), bundle_locations_(bundle_locations), cluster_resources_(cluster_resources) {} @@ -98,7 +102,7 @@ class ScheduleContext { // The locations of existing bundles for this placement group. const absl::optional> bundle_locations_; // The available resources of all nodes. - const absl::flat_hash_map &cluster_resources_; + const absl::flat_hash_map &cluster_resources_; }; class GcsScheduleStrategy { @@ -168,8 +172,10 @@ enum class LeasingState { /// status. class LeaseStatusTracker { public: - LeaseStatusTracker(std::shared_ptr placement_group, - std::vector> &unplaced_bundles); + LeaseStatusTracker( + std::shared_ptr placement_group, + const std::vector> &unplaced_bundles, + const ScheduleMap &schedule_map); ~LeaseStatusTracker() = default; /// Indicate the tracker that prepare requests are sent to a specific node. @@ -239,6 +245,16 @@ class LeaseStatusTracker { /// \return Location of bundles that failed to commit resources on a node. const std::shared_ptr &GetUnCommittedBundleLocations() const; + /// This method returns bundle locations that success to commit resources. + /// + /// \return Location of bundles that success to commit resources on a node. + const std::shared_ptr &GetCommittedBundleLocations() const; + + /// This method returns bundle locations. + /// + /// \return Location of bundles. + const std::shared_ptr &GetBundleLocations() const; + /// Return the leasing state. /// /// \return Leasing state. @@ -276,6 +292,9 @@ class LeaseStatusTracker { /// Location of bundles that commit requests failed. std::shared_ptr uncommitted_bundle_locations_; + /// Location of bundles that committed requests success. + std::shared_ptr committed_bundle_locations_; + /// The leasing stage. This is used to know the state of current leasing context. LeasingState leasing_state_ = LeasingState::PREPARING; @@ -288,6 +307,9 @@ class LeaseStatusTracker { /// Bundles to schedule. std::vector> bundles_to_schedule_; + + /// Location of bundles. + std::shared_ptr bundle_locations_; }; /// A data structure that helps fast bundle location lookup. @@ -364,7 +386,7 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { boost::asio::io_context &io_context, std::shared_ptr gcs_table_storage, const GcsNodeManager &gcs_node_manager, GcsResourceManager &gcs_resource_manager, - ReserveResourceClientFactoryFn lease_client_factory = nullptr); + std::shared_ptr raylet_client_pool); virtual ~GcsPlacementGroupScheduler() = default; @@ -492,14 +514,20 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { void DestroyPlacementGroupCommittedBundleResources( const PlacementGroupID &placement_group_id); + /// Acquire the bundle resources from the cluster resources. + void AcquireBundleResources(const std::shared_ptr &bundle_locations); + + /// Return the bundle resources to the cluster resources. + void ReturnBundleResources(const std::shared_ptr &bundle_locations); + /// Generate schedule context. std::unique_ptr GetScheduleContext( const PlacementGroupID &placement_group_id); /// A timer that ticks every cancel resource failure milliseconds. boost::asio::deadline_timer return_timer_; - /// Used to update placement group information upon creation, deletion, etc. + /// Used to update placement group information upon creation, deletion, etc. std::shared_ptr gcs_table_storage_; /// Reference of GcsNodeManager. @@ -508,13 +536,6 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { /// Reference of GcsResourceManager. GcsResourceManager &gcs_resource_manager_; - /// The cached node clients which are used to communicate with raylet to lease workers. - absl::flat_hash_map> - remote_lease_clients_; - - /// Factory for producing new clients to request leases from remote nodes. - ReserveResourceClientFactoryFn lease_client_factory_; - /// A vector to store all the schedule strategy. std::vector> scheduler_strategies_; @@ -525,6 +546,9 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { absl::flat_hash_map> placement_group_leasing_in_progress_; + /// The cached raylet clients used to communicate with raylets. + std::shared_ptr raylet_client_pool_; + /// The nodes which are releasing unused bundles. absl::flat_hash_set nodes_of_releasing_unused_bundles_; }; diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.cc b/src/ray/gcs/gcs_server/gcs_resource_manager.cc index eeb5630f7..7357eeaaf 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.cc @@ -17,42 +17,215 @@ namespace ray { namespace gcs { -const absl::flat_hash_map &GcsResourceManager::GetClusterResources() - const { - return cluster_resources_; +GcsResourceManager::GcsResourceManager( + std::shared_ptr gcs_pub_sub, + std::shared_ptr gcs_table_storage) + : gcs_pub_sub_(gcs_pub_sub), gcs_table_storage_(gcs_table_storage) {} + +void GcsResourceManager::HandleGetResources(const rpc::GetResourcesRequest &request, + rpc::GetResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) { + NodeID node_id = NodeID::FromBinary(request.node_id()); + auto iter = cluster_resources_.find(node_id); + if (iter != cluster_resources_.end()) { + for (const auto &resource : iter->second.items()) { + (*reply->mutable_resources())[resource.first] = resource.second; + } + } + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); + ++counts_[CountType::GET_RESOURCES_REQUEST]; } -void GcsResourceManager::UpdateResources(const NodeID &node_id, - const ResourceSet &resources) { - cluster_resources_[node_id] = resources; +void GcsResourceManager::HandleUpdateResources( + const rpc::UpdateResourcesRequest &request, rpc::UpdateResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) { + NodeID node_id = NodeID::FromBinary(request.node_id()); + RAY_LOG(DEBUG) << "Updating resources, node id = " << node_id; + auto iter = cluster_resources_.find(node_id); + std::unordered_map to_be_updated_resources; + for (const auto &entry : request.resources()) { + to_be_updated_resources.emplace(entry.first, entry.second.resource_capacity()); + } + + if (iter != cluster_resources_.end()) { + for (const auto &entry : request.resources()) { + (*iter->second.mutable_items())[entry.first] = entry.second; + } + UpdateResourceCapacity(node_id, to_be_updated_resources); + auto on_done = [this, node_id, to_be_updated_resources, reply, + send_reply_callback](const Status &status) { + RAY_CHECK_OK(status); + rpc::NodeResourceChange node_resource_change; + node_resource_change.set_node_id(node_id.Binary()); + for (const auto &it : to_be_updated_resources) { + const auto &resource_name = it.first; + const auto &resource_capacity = it.second; + auto &node_updated_resources = + (*node_resource_change.mutable_updated_resources()); + node_updated_resources[resource_name] = resource_capacity; + } + RAY_CHECK_OK(gcs_pub_sub_->Publish(NODE_RESOURCE_CHANNEL, node_id.Hex(), + node_resource_change.SerializeAsString(), + nullptr)); + + GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); + RAY_LOG(DEBUG) << "Finished updating resources, node id = " << node_id; + }; + + RAY_CHECK_OK( + gcs_table_storage_->NodeResourceTable().Put(node_id, iter->second, on_done)); + } else { + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::Invalid("Node is not exist.")); + RAY_LOG(ERROR) << "Failed to update resources as node " << node_id + << " is not registered."; + } + ++counts_[CountType::UPDATE_RESOURCES_REQUEST]; } -void GcsResourceManager::RemoveResources(const NodeID &node_id) { +void GcsResourceManager::HandleDeleteResources( + const rpc::DeleteResourcesRequest &request, rpc::DeleteResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) { + NodeID node_id = NodeID::FromBinary(request.node_id()); + RAY_LOG(DEBUG) << "Deleting node resources, node id = " << node_id; + auto resource_names = VectorFromProtobuf(request.resource_name_list()); + auto iter = cluster_resources_.find(node_id); + if (iter != cluster_resources_.end()) { + DeleteResources(node_id, resource_names); + + for (const auto &resource_name : resource_names) { + RAY_IGNORE_EXPR(iter->second.mutable_items()->erase(resource_name)); + } + auto on_done = [this, node_id, resource_names, reply, + send_reply_callback](const Status &status) { + RAY_CHECK_OK(status); + rpc::NodeResourceChange node_resource_change; + node_resource_change.set_node_id(node_id.Binary()); + for (const auto &resource_name : resource_names) { + node_resource_change.add_deleted_resources(resource_name); + } + RAY_CHECK_OK(gcs_pub_sub_->Publish(NODE_RESOURCE_CHANNEL, node_id.Hex(), + node_resource_change.SerializeAsString(), + nullptr)); + + GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); + }; + RAY_CHECK_OK( + gcs_table_storage_->NodeResourceTable().Put(node_id, iter->second, on_done)); + } else { + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); + RAY_LOG(DEBUG) << "Finished deleting node resources, node id = " << node_id; + } + ++counts_[CountType::DELETE_RESOURCES_REQUEST]; +} + +void GcsResourceManager::HandleGetAllAvailableResources( + const rpc::GetAllAvailableResourcesRequest &request, + rpc::GetAllAvailableResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) { + for (const auto &iter : cluster_scheduling_resources_) { + rpc::AvailableResources resource; + resource.set_node_id(iter.first.Binary()); + for (const auto &res : iter.second.GetAvailableResources().GetResourceAmountMap()) { + (*resource.mutable_resources_available())[res.first] = res.second.ToDouble(); + } + reply->add_resources_list()->CopyFrom(resource); + } + GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); + ++counts_[CountType::GET_ALL_AVAILABLE_RESOURCES_REQUEST]; +} + +void GcsResourceManager::Initialize(const GcsInitData &gcs_init_data) { + const auto &nodes = gcs_init_data.Nodes(); + for (auto &entry : gcs_init_data.ClusterResources()) { + const auto &iter = nodes.find(entry.first); + if (iter->second.state() == rpc::GcsNodeInfo::ALIVE) { + cluster_resources_[entry.first] = entry.second; + } + } +} + +const absl::flat_hash_map + &GcsResourceManager::GetClusterResources() const { + return cluster_scheduling_resources_; +} + +void GcsResourceManager::SetAvailableResources(const NodeID &node_id, + const ResourceSet &resources) { + cluster_scheduling_resources_[node_id].SetAvailableResources(ResourceSet(resources)); +} + +void GcsResourceManager::UpdateResourceCapacity( + const NodeID &node_id, + const std::unordered_map &changed_resources) { + auto iter = cluster_scheduling_resources_.find(node_id); + if (iter != cluster_scheduling_resources_.end()) { + SchedulingResources &scheduling_resources = iter->second; + for (const auto &entry : changed_resources) { + scheduling_resources.UpdateResourceCapacity(entry.first, entry.second); + } + } else { + cluster_scheduling_resources_.emplace( + node_id, SchedulingResources(ResourceSet(changed_resources))); + } +} + +void GcsResourceManager::DeleteResources( + const NodeID &node_id, const std::vector &deleted_resources) { + auto iter = cluster_scheduling_resources_.find(node_id); + if (iter != cluster_scheduling_resources_.end()) { + for (auto &resource_name : deleted_resources) { + iter->second.DeleteResource(resource_name); + } + } +} + +void GcsResourceManager::OnNodeAdd(const NodeID &node_id) { + // Add an empty resources for this node. + cluster_resources_.emplace(node_id, rpc::ResourceMap()); +} + +void GcsResourceManager::OnNodeDead(const NodeID &node_id) { cluster_resources_.erase(node_id); + cluster_scheduling_resources_.erase(node_id); } bool GcsResourceManager::AcquireResources(const NodeID &node_id, const ResourceSet &required_resources) { - auto iter = cluster_resources_.find(node_id); - // - RAY_CHECK(iter != cluster_resources_.end()) << "Node " << node_id << " not exist."; - if (!required_resources.IsSubset(iter->second)) { - return false; - } - iter->second.SubtractResourcesStrict(required_resources); - return true; -} - -bool GcsResourceManager::ReleaseResources(const NodeID &node_id, - const ResourceSet &acquired_resources) { - auto iter = cluster_resources_.find(node_id); - if (iter != cluster_resources_.end()) { - iter->second.AddResources(acquired_resources); + auto iter = cluster_scheduling_resources_.find(node_id); + if (iter != cluster_scheduling_resources_.end()) { + if (!required_resources.IsSubset(iter->second.GetAvailableResources())) { + return false; + } + iter->second.Acquire(required_resources); } // If node dead, we will not find the node. This is a normal scenario, so it returns // true. return true; } +bool GcsResourceManager::ReleaseResources(const NodeID &node_id, + const ResourceSet &acquired_resources) { + auto iter = cluster_scheduling_resources_.find(node_id); + if (iter != cluster_scheduling_resources_.end()) { + iter->second.Release(acquired_resources); + } + // If node dead, we will not find the node. This is a normal scenario, so it returns + // true. + return true; +} + +std::string GcsResourceManager::DebugString() const { + std::ostringstream stream; + stream << "GcsResourceManager: {GetResources request count: " + << counts_[CountType::GET_RESOURCES_REQUEST] + << ", GetAllAvailableResources request count" + << counts_[CountType::GET_ALL_AVAILABLE_RESOURCES_REQUEST] + << ", UpdateResources request count: " + << counts_[CountType::UPDATE_RESOURCES_REQUEST] + << ", DeleteResources request count: " + << counts_[CountType::DELETE_RESOURCES_REQUEST] << "}"; + return stream.str(); +} + } // namespace gcs } // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.h b/src/ray/gcs/gcs_server/gcs_resource_manager.h index 0a38d07ac..eda9ced4d 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.h @@ -14,33 +14,77 @@ #pragma once #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "ray/common/id.h" #include "ray/common/task/scheduling_resources.h" +#include "ray/gcs/accessor.h" +#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/rpc/client_call.h" +#include "ray/rpc/gcs_server/gcs_rpc_server.h" +#include "src/ray/protobuf/gcs.pb.h" namespace ray { namespace gcs { /// Gcs resource manager interface. -/// It is used for actor and placement group scheduling. -/// Non-thread safe. -class GcsResourceManagerInterface { +/// It is responsible for handing node resource related rpc requests and it is used for +/// actor and placement group scheduling. It obtains the available resources of nodes +/// through heartbeat reporting. Non-thread safe. +class GcsResourceManager : public rpc::NodeResourceInfoHandler { public: - virtual ~GcsResourceManagerInterface() {} + /// Create a GcsResourceManager. + /// + /// \param gcs_pub_sub GCS message publisher. + /// \param gcs_table_storage GCS table external storage accessor. + explicit GcsResourceManager(std::shared_ptr gcs_pub_sub, + std::shared_ptr gcs_table_storage); + + virtual ~GcsResourceManager() {} + + /// Handle get resource rpc request. + void HandleGetResources(const rpc::GetResourcesRequest &request, + rpc::GetResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + + /// Handle update resource rpc request. + void HandleUpdateResources(const rpc::UpdateResourcesRequest &request, + rpc::UpdateResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + + /// Handle delete resource rpc request. + void HandleDeleteResources(const rpc::DeleteResourcesRequest &request, + rpc::DeleteResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + + /// Handle get available resources of all nodes. + void HandleGetAllAvailableResources( + const rpc::GetAllAvailableResourcesRequest &request, + rpc::GetAllAvailableResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) override; /// Get the resources of all nodes in the cluster. /// /// \return The resources of all nodes in the cluster. - virtual const absl::flat_hash_map &GetClusterResources() const = 0; + const absl::flat_hash_map &GetClusterResources() const; - /// Update the resources of the specified node. + /// Handle a node registration. + /// + /// \param node_id The specified node id. + void OnNodeAdd(const NodeID &node_id); + + /// Handle a node death. + /// + /// \param node_id The specified node id. + void OnNodeDead(const NodeID &node_id); + + /// Set the available resources of the specified node. /// /// \param node_id Id of a node. - /// \param resources Resources of a node. - virtual void UpdateResources(const NodeID &node_id, const ResourceSet &resources) = 0; - - /// Remove the resources of the specified node. - /// - /// \param node_id Id of a node. - virtual void RemoveResources(const NodeID &node_id) = 0; + /// \param resources Available resources of a node. + void SetAvailableResources(const NodeID &node_id, const ResourceSet &resources); /// Acquire resources from the specified node. It will deduct directly from the node /// resources. @@ -48,8 +92,7 @@ class GcsResourceManagerInterface { /// \param node_id Id of a node. /// \param required_resources Resources to apply for. /// \return True if acquire resources successfully. False otherwise. - virtual bool AcquireResources(const NodeID &node_id, - const ResourceSet &required_resources) = 0; + bool AcquireResources(const NodeID &node_id, const ResourceSet &required_resources); /// Release the resources of the specified node. It will be added directly to the node /// resources. @@ -57,29 +100,50 @@ class GcsResourceManagerInterface { /// \param node_id Id of a node. /// \param acquired_resources Resources to release. /// \return True if release resources successfully. False otherwise. - virtual bool ReleaseResources(const NodeID &node_id, - const ResourceSet &acquired_resources) = 0; -}; - -/// Gcs resource manager implementation. It obtains the available resources of nodes -/// through heartbeat reporting. Non-thread safe. -class GcsResourceManager : public GcsResourceManagerInterface { - public: - virtual ~GcsResourceManager() = default; - - const absl::flat_hash_map &GetClusterResources() const; - - void UpdateResources(const NodeID &node_id, const ResourceSet &resources); - - void RemoveResources(const NodeID &node_id); - - bool AcquireResources(const NodeID &node_id, const ResourceSet &required_resources); - bool ReleaseResources(const NodeID &node_id, const ResourceSet &acquired_resources); + /// Initialize with the gcs tables data synchronously. + /// This should be called when GCS server restarts after a failure. + /// + /// \param gcs_init_data. + void Initialize(const GcsInitData &gcs_init_data); + + std::string DebugString() const; + + /// Update the total resources and available resources of the specified node. + /// + /// \param node_id Id of a node. + /// \param changed_resources Changed resources of a node. + void UpdateResourceCapacity( + const NodeID &node_id, + const std::unordered_map &changed_resources); + private: - /// Map from node id to the resources of the node. - absl::flat_hash_map cluster_resources_; + /// Delete the scheduling resources of the specified node. + /// + /// \param node_id Id of a node. + /// \param deleted_resources Deleted resources of a node. + void DeleteResources(const NodeID &node_id, + const std::vector &deleted_resources); + + /// A publisher for publishing gcs messages. + std::shared_ptr gcs_pub_sub_; + /// Storage for GCS tables. + std::shared_ptr gcs_table_storage_; + /// Cluster resources. + absl::flat_hash_map cluster_resources_; + /// Map from node id to the scheduling resources of the node. + absl::flat_hash_map cluster_scheduling_resources_; + + /// Debug info. + enum CountType { + GET_RESOURCES_REQUEST = 0, + UPDATE_RESOURCES_REQUEST = 1, + DELETE_RESOURCES_REQUEST = 2, + GET_ALL_AVAILABLE_RESOURCES_REQUEST = 3, + CountType_MAX = 4, + }; + uint64_t counts_[CountType::CountType_MAX] = {0}; }; } // namespace gcs diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index a5874ceef..bf8ca289d 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -35,7 +35,9 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, main_service_(main_service), rpc_server_(config.grpc_server_name, config.grpc_server_port, config.grpc_server_thread_num), - client_call_manager_(main_service) {} + client_call_manager_(main_service), + raylet_client_pool_( + std::make_shared(client_call_manager_)) {} GcsServer::~GcsServer() { Stop(); } @@ -66,11 +68,14 @@ void GcsServer::Start() { void GcsServer::DoStart(const GcsInitData &gcs_init_data) { // Init gcs resource manager. - InitGcsResourceManager(); + InitGcsResourceManager(gcs_init_data); // Init gcs node manager. InitGcsNodeManager(gcs_init_data); + // Init gcs heartbeat manager. + InitGcsHeartbeatManager(gcs_init_data); + // Init gcs job manager. InitGcsJobManager(); @@ -101,11 +106,11 @@ void GcsServer::DoStart(const GcsInitData &gcs_init_data) { // Store gcs rpc server address in redis. StoreGcsServerAddressInRedis(); - // Only after the rpc_server_ is running can the node failure - // detector be run. Otherwise the node failure detector will mistake + // Only after the rpc_server_ is running can the heartbeat manager + // be run. Otherwise the node failure detector will mistake // some living nodes as dead as the timer inside node failure // detector is already run. - gcs_node_manager_->StartNodeFailureDetector(); + gcs_heartbeat_manager_->Start(); // Print debug info periodically. PrintDebugInfo(); @@ -119,10 +124,7 @@ void GcsServer::Stop() { // Shutdown the rpc server rpc_server_.Shutdown(); - node_manager_io_service_.stop(); - if (node_manager_io_service_thread_->joinable()) { - node_manager_io_service_thread_->join(); - } + gcs_heartbeat_manager_->Stop(); is_stopped_ = true; RAY_LOG(INFO) << "GCS server stopped."; @@ -131,14 +133,8 @@ void GcsServer::Stop() { void GcsServer::InitGcsNodeManager(const GcsInitData &gcs_init_data) { RAY_CHECK(redis_gcs_client_ && gcs_table_storage_ && gcs_pub_sub_); - node_manager_io_service_thread_.reset(new std::thread([this] { - /// The asio work to keep node_manager_io_service_ alive. - boost::asio::io_service::work node_manager_io_service_work_(node_manager_io_service_); - node_manager_io_service_.run(); - })); gcs_node_manager_ = std::make_shared( - main_service_, node_manager_io_service_, gcs_pub_sub_, gcs_table_storage_, - gcs_resource_manager_); + main_service_, gcs_pub_sub_, gcs_table_storage_, gcs_resource_manager_); // Initialize by gcs tables data. gcs_node_manager_->Initialize(gcs_init_data); // Register service. @@ -147,8 +143,33 @@ void GcsServer::InitGcsNodeManager(const GcsInitData &gcs_init_data) { rpc_server_.RegisterService(*node_info_service_); } -void GcsServer::InitGcsResourceManager() { - gcs_resource_manager_ = std::make_shared(); +void GcsServer::InitGcsHeartbeatManager(const GcsInitData &gcs_init_data) { + RAY_CHECK(gcs_node_manager_); + gcs_heartbeat_manager_ = std::make_shared( + heartbeat_manager_io_service_, /*on_node_death_callback=*/ + [this](const NodeID &node_id) { + main_service_.post( + [this, node_id] { return gcs_node_manager_->OnNodeFailure(node_id); }); + }); + for (const auto &node : gcs_init_data.Nodes()) { + gcs_heartbeat_manager_->AddNode(node.first); + } + // Register service. + heartbeat_info_service_.reset(new rpc::HeartbeatInfoGrpcService( + heartbeat_manager_io_service_, *gcs_heartbeat_manager_)); + rpc_server_.RegisterService(*heartbeat_info_service_); +} + +void GcsServer::InitGcsResourceManager(const GcsInitData &gcs_init_data) { + RAY_CHECK(gcs_table_storage_ && gcs_pub_sub_); + gcs_resource_manager_ = + std::make_shared(gcs_pub_sub_, gcs_table_storage_); + // Initialize by gcs tables data. + gcs_resource_manager_->Initialize(gcs_init_data); + // Register service. + node_resource_info_service_.reset( + new rpc::NodeResourceInfoGrpcService(main_service_, *gcs_resource_manager_)); + rpc_server_.RegisterService(*node_resource_info_service_); } void GcsServer::InitGcsJobManager() { @@ -175,13 +196,7 @@ void GcsServer::InitGcsActorManager(const GcsInitData &gcs_init_data) { [this](std::shared_ptr actor) { gcs_actor_manager_->OnActorCreationSuccess(std::move(actor)); }, - /*lease_client_factory=*/ - [this](const rpc::Address &address) { - auto node_manager_worker_client = rpc::NodeManagerWorkerClient::make( - address.ip_address(), address.port(), client_call_manager_); - return std::make_shared( - std::move(node_manager_worker_client)); - }, + raylet_client_pool_, /*client_factory=*/ [this](const rpc::Address &address) { return std::make_shared(address, client_call_manager_); @@ -194,6 +209,7 @@ void GcsServer::InitGcsActorManager(const GcsInitData &gcs_init_data) { [this](const rpc::Address &address) { return std::make_shared(address, client_call_manager_); }); + // Initialize by gcs tables data. gcs_actor_manager_->Initialize(gcs_init_data); // Register service. @@ -206,13 +222,7 @@ void GcsServer::InitGcsPlacementGroupManager(const GcsInitData &gcs_init_data) { RAY_CHECK(gcs_table_storage_ && gcs_node_manager_); auto scheduler = std::make_shared( main_service_, gcs_table_storage_, *gcs_node_manager_, *gcs_resource_manager_, - /*lease_client_factory=*/ - [this](const rpc::Address &address) { - auto node_manager_worker_client = rpc::NodeManagerWorkerClient::make( - address.ip_address(), address.port(), client_call_manager_); - return std::make_shared( - std::move(node_manager_worker_client)); - }); + raylet_client_pool_); gcs_placement_group_manager_ = std::make_shared( main_service_, scheduler, gcs_table_storage_, *gcs_node_manager_); @@ -285,6 +295,7 @@ void GcsServer::InstallEventListeners() { // placement groups and the pending actors. gcs_placement_group_manager_->SchedulePendingPlacementGroups(); gcs_actor_manager_->SchedulePendingActors(); + gcs_heartbeat_manager_->AddNode(NodeID::FromBinary(node->node_id())); }); gcs_node_manager_->AddNodeRemovedListener( [this](std::shared_ptr node) { @@ -293,7 +304,7 @@ void GcsServer::InstallEventListeners() { // node is removed from the GCS. gcs_placement_group_manager_->OnNodeDead(node_id); gcs_actor_manager_->OnNodeDead(node_id); - gcs_resource_manager_->RemoveResources(node_id); + raylet_client_pool_->Disconnect(NodeID::FromBinary(node->node_id())); }); // Install worker event listener. diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 8d9d55c4b..a2082539f 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -14,6 +14,7 @@ #pragma once +#include "ray/gcs/gcs_server/gcs_heartbeat_manager.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_object_manager.h" #include "ray/gcs/gcs_server/gcs_redis_failure_detector.h" @@ -23,6 +24,7 @@ #include "ray/gcs/redis_gcs_client.h" #include "ray/rpc/client_call.h" #include "ray/rpc/gcs_server/gcs_rpc_server.h" +#include "ray/rpc/node_manager/node_manager_client_pool.h" namespace ray { namespace gcs { @@ -78,8 +80,11 @@ class GcsServer { /// Initialize gcs node manager. void InitGcsNodeManager(const GcsInitData &gcs_init_data); + /// Initialize gcs heartbeat manager. + void InitGcsHeartbeatManager(const GcsInitData &gcs_init_data); + /// Initialize gcs resource manager. - void InitGcsResourceManager(); + void InitGcsResourceManager(const GcsInitData &gcs_init_data); /// Initialize gcs job manager. void InitGcsJobManager(); @@ -123,18 +128,21 @@ class GcsServer { GcsServerConfig config_; /// The main io service to drive event posted from grpc threads. boost::asio::io_context &main_service_; - /// The io service used by node manager in case of node failure detector being blocked - /// by main thread. - boost::asio::io_service node_manager_io_service_; - std::unique_ptr node_manager_io_service_thread_; + /// The io service used by heartbeat manager in case of node failure detector being + /// blocked by main thread. + boost::asio::io_service heartbeat_manager_io_service_; /// The grpc server rpc::GrpcServer rpc_server_; /// The `ClientCallManager` object that is shared by all `NodeManagerWorkerClient`s. rpc::ClientCallManager client_call_manager_; + /// Node manager client pool + std::shared_ptr raylet_client_pool_; /// The gcs resource manager. std::shared_ptr gcs_resource_manager_; /// The gcs node manager. std::shared_ptr gcs_node_manager_; + /// The heartbeat manager. + std::shared_ptr gcs_heartbeat_manager_; /// The gcs redis failure detector. std::shared_ptr gcs_redis_failure_detector_; /// The gcs actor manager @@ -148,6 +156,10 @@ class GcsServer { std::unique_ptr actor_info_service_; /// Node info handler and service std::unique_ptr node_info_service_; + /// Node resource info handler and service + std::unique_ptr node_resource_info_service_; + /// Heartbeat info handler and service + std::unique_ptr heartbeat_info_service_; /// Object info handler and service std::unique_ptr gcs_object_manager_; std::unique_ptr object_info_service_; diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index a89d8ead0..04e527dd5 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -61,7 +61,7 @@ int main(int argc, char *argv[]) { RayConfig::instance().initialize(config_map); const ray::stats::TagsType global_tags = { {ray::stats::ComponentKey, "gcs_server"}, - {ray::stats::VersionKey, "1.1.0.dev0"}, + {ray::stats::VersionKey, "1.2.0.dev0"}, {ray::stats::NodeAddressKey, node_ip_address}}; ray::stats::Init(global_tags, metrics_agent_port); diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_server/gcs_table_storage.cc index 41a914272..5eea83484 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_server/gcs_table_storage.cc @@ -130,7 +130,7 @@ template class GcsTable; template class GcsTable; template class GcsTable; template class GcsTable; -template class GcsTable; +template class GcsTable; template class GcsTable; template class GcsTable; template class GcsTable; diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.h b/src/ray/gcs/gcs_server/gcs_table_storage.h index 6378c161b..4460d150a 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.h +++ b/src/ray/gcs/gcs_server/gcs_table_storage.h @@ -26,7 +26,6 @@ namespace gcs { using rpc::ActorTableData; using rpc::ErrorTableData; using rpc::GcsNodeInfo; -using rpc::HeartbeatBatchTableData; using rpc::HeartbeatTableData; using rpc::JobTableData; using rpc::ObjectLocationInfo; @@ -35,6 +34,7 @@ using rpc::PlacementGroupTableData; using rpc::ProfileTableData; using rpc::ResourceMap; using rpc::ResourceTableData; +using rpc::ResourceUsageBatchData; using rpc::ScheduleData; using rpc::StoredConfig; using rpc::TaskLeaseData; @@ -255,11 +255,11 @@ class GcsPlacementGroupScheduleTable : public GcsTable { +class GcsResourceUsageBatchTable : public GcsTable { public: - explicit GcsHeartbeatBatchTable(std::shared_ptr &store_client) + explicit GcsResourceUsageBatchTable(std::shared_ptr &store_client) : GcsTable(store_client) { - table_name_ = TablePrefix_Name(TablePrefix::HEARTBEAT_BATCH); + table_name_ = TablePrefix_Name(TablePrefix::RESOURCE_USAGE_BATCH); } }; @@ -348,9 +348,9 @@ class GcsTableStorage { return *heartbeat_table_; } - GcsHeartbeatBatchTable &HeartbeatBatchTable() { - RAY_CHECK(heartbeat_batch_table_ != nullptr); - return *heartbeat_batch_table_; + GcsResourceUsageBatchTable &HeartbeatBatchTable() { + RAY_CHECK(resource_usage_batch_table_ != nullptr); + return *resource_usage_batch_table_; } GcsProfileTable &ProfileTable() { @@ -381,7 +381,7 @@ class GcsTableStorage { std::unique_ptr node_resource_table_; std::unique_ptr placement_group_schedule_table_; std::unique_ptr heartbeat_table_; - std::unique_ptr heartbeat_batch_table_; + std::unique_ptr resource_usage_batch_table_; std::unique_ptr profile_table_; std::unique_ptr worker_table_; std::unique_ptr system_config_table_; @@ -408,7 +408,7 @@ class RedisGcsTableStorage : public GcsTableStorage { heartbeat_table_.reset(new GcsHeartbeatTable(store_client_)); placement_group_schedule_table_.reset( new GcsPlacementGroupScheduleTable(store_client_)); - heartbeat_batch_table_.reset(new GcsHeartbeatBatchTable(store_client_)); + resource_usage_batch_table_.reset(new GcsResourceUsageBatchTable(store_client_)); profile_table_.reset(new GcsProfileTable(store_client_)); worker_table_.reset(new GcsWorkerTable(store_client_)); system_config_table_.reset(new GcsInternalConfigTable(store_client_)); @@ -434,7 +434,7 @@ class InMemoryGcsTableStorage : public GcsTableStorage { placement_group_schedule_table_.reset( new GcsPlacementGroupScheduleTable(store_client_)); heartbeat_table_.reset(new GcsHeartbeatTable(store_client_)); - heartbeat_batch_table_.reset(new GcsHeartbeatBatchTable(store_client_)); + resource_usage_batch_table_.reset(new GcsResourceUsageBatchTable(store_client_)); profile_table_.reset(new GcsProfileTable(store_client_)); worker_table_.reset(new GcsWorkerTable(store_client_)); system_config_table_.reset(new GcsInternalConfigTable(store_client_)); diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc index 6be265678..4ddba0627 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc @@ -19,7 +19,6 @@ #include "ray/gcs/test/gcs_test_util.h" namespace ray { - class GcsActorSchedulerTest : public ::testing::Test { public: void SetUp() override { @@ -27,13 +26,14 @@ class GcsActorSchedulerTest : public ::testing::Test { worker_client_ = std::make_shared(); gcs_pub_sub_ = std::make_shared(redis_client_); gcs_table_storage_ = std::make_shared(redis_client_); - gcs_resource_manager_ = std::make_shared(); - gcs_node_manager_ = - std::make_shared(io_service_, io_service_, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); + gcs_node_manager_ = std::make_shared( + io_service_, gcs_pub_sub_, gcs_table_storage_, gcs_resource_manager_); store_client_ = std::make_shared(io_service_); gcs_actor_table_ = std::make_shared(store_client_); + raylet_client_pool_ = std::make_shared( + [this](const rpc::Address &addr) { return raylet_client_; }); gcs_actor_scheduler_ = std::make_shared( io_service_, *gcs_actor_table_, *gcs_node_manager_, gcs_pub_sub_, /*schedule_failure_handler=*/ @@ -44,8 +44,7 @@ class GcsActorSchedulerTest : public ::testing::Test { [this](std::shared_ptr actor) { success_actors_.emplace_back(std::move(actor)); }, - /*lease_client_factory=*/ - [this](const rpc::Address &address) { return raylet_client_; }, + raylet_client_pool_, /*client_factory=*/ [this](const rpc::Address &address) { return worker_client_; }); } @@ -64,6 +63,7 @@ class GcsActorSchedulerTest : public ::testing::Test { std::shared_ptr gcs_pub_sub_; std::shared_ptr gcs_table_storage_; std::shared_ptr redis_client_; + std::shared_ptr raylet_client_pool_; }; TEST_F(GcsActorSchedulerTest, TestScheduleFailedWithZeroNode) { diff --git a/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc index 17ea70e8d..a904512ac 100644 --- a/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc @@ -23,6 +23,7 @@ class GcsNodeManagerTest : public ::testing::Test { public: GcsNodeManagerTest() { gcs_pub_sub_ = std::make_shared(redis_client_); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); } protected: @@ -34,23 +35,23 @@ class GcsNodeManagerTest : public ::testing::Test { TEST_F(GcsNodeManagerTest, TestManagement) { boost::asio::io_service io_service; - gcs::GcsNodeManager node_manager(io_service, io_service, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs::GcsNodeManager node_manager(io_service, gcs_pub_sub_, gcs_table_storage_, + gcs_resource_manager_); // Test Add/Get/Remove functionality. auto node = Mocker::GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); node_manager.AddNode(node); - ASSERT_EQ(node, node_manager.GetNode(node_id).value()); + ASSERT_EQ(node, node_manager.GetAliveNode(node_id).value()); node_manager.RemoveNode(node_id); - ASSERT_TRUE(!node_manager.GetNode(node_id).has_value()); + ASSERT_TRUE(!node_manager.GetAliveNode(node_id).has_value()); } TEST_F(GcsNodeManagerTest, TestListener) { boost::asio::io_service io_service; - gcs::GcsNodeManager node_manager(io_service, io_service, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs::GcsNodeManager node_manager(io_service, gcs_pub_sub_, gcs_table_storage_, + gcs_resource_manager_); // Test AddNodeAddedListener. int node_count = 1000; std::vector> added_nodes; diff --git a/src/ray/gcs/gcs_server/test/gcs_object_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_object_manager_test.cc index 6d4484c7d..15f96a6a8 100644 --- a/src/ray/gcs/gcs_server/test/gcs_object_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_object_manager_test.cc @@ -54,10 +54,9 @@ class GcsObjectManagerTest : public ::testing::Test { public: void SetUp() override { gcs_table_storage_ = std::make_shared(io_service_); - gcs_resource_manager_ = std::make_shared(); - gcs_node_manager_ = - std::make_shared(io_service_, io_service_, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); + gcs_node_manager_ = std::make_shared( + io_service_, gcs_pub_sub_, gcs_table_storage_, gcs_resource_manager_); gcs_object_manager_ = std::make_shared( gcs_table_storage_, gcs_pub_sub_, *gcs_node_manager_); GenTestData(); diff --git a/src/ray/gcs/gcs_server/test/gcs_placement_group_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_placement_group_manager_test.cc index 80b83ea81..e74b5fe1b 100644 --- a/src/ray/gcs/gcs_server/test/gcs_placement_group_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_placement_group_manager_test.cc @@ -68,10 +68,9 @@ class GcsPlacementGroupManagerTest : public ::testing::Test { : mock_placement_group_scheduler_(new MockPlacementGroupScheduler()) { gcs_pub_sub_ = std::make_shared(redis_client_); gcs_table_storage_ = std::make_shared(io_service_); - gcs_resource_manager_ = std::make_shared(); - gcs_node_manager_ = - std::make_shared(io_service_, io_service_, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); + gcs_node_manager_ = std::make_shared( + io_service_, gcs_pub_sub_, gcs_table_storage_, gcs_resource_manager_); gcs_placement_group_manager_.reset( new gcs::GcsPlacementGroupManager(io_service_, mock_placement_group_scheduler_, gcs_table_storage_, *gcs_node_manager_)); diff --git a/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc index cc243582c..3bf5923c9 100644 --- a/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc @@ -35,21 +35,20 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { })); for (int index = 0; index < 3; ++index) { - raylet_clients_.push_back( - std::make_shared()); + raylet_clients_.push_back(std::make_shared()); } gcs_table_storage_ = std::make_shared(io_service_); gcs_pub_sub_ = std::make_shared(redis_client_); - gcs_resource_manager_ = std::make_shared(); - gcs_node_manager_ = - std::make_shared(io_service_, io_service_, gcs_pub_sub_, - gcs_table_storage_, gcs_resource_manager_); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); + gcs_node_manager_ = std::make_shared( + io_service_, gcs_pub_sub_, gcs_table_storage_, gcs_resource_manager_); gcs_table_storage_ = std::make_shared(io_service_); store_client_ = std::make_shared(io_service_); + raylet_client_pool_ = std::make_shared( + [this](const rpc::Address &addr) { return raylet_clients_[addr.port()]; }); scheduler_ = std::make_shared( io_service_, gcs_table_storage_, *gcs_node_manager_, *gcs_resource_manager_, - /*lease_client_fplacement_groupy=*/ - [this](const rpc::Address &address) { return raylet_clients_[address.port()]; }); + raylet_client_pool_); } void TearDown() override { @@ -99,11 +98,11 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { void AddNode(const std::shared_ptr &node, int cpu_num = 10) { gcs_node_manager_->AddNode(node); - rpc::HeartbeatTableData heartbeat; - heartbeat.set_node_id(node->node_id()); - (*heartbeat.mutable_resources_available())["CPU"] = cpu_num; + rpc::ResourcesData resource; + resource.set_node_id(node->node_id()); + (*resource.mutable_resources_available())["CPU"] = cpu_num; gcs_node_manager_->UpdateNodeRealtimeResources(NodeID::FromBinary(node->node_id()), - heartbeat); + resource); } void ScheduleFailedWithZeroNodeTest(rpc::PlacementStrategy strategy) { @@ -204,7 +203,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { boost::asio::io_service io_service_; std::shared_ptr store_client_; - std::vector> raylet_clients_; + std::vector> raylet_clients_; std::shared_ptr gcs_resource_manager_; std::shared_ptr gcs_node_manager_; std::shared_ptr scheduler_; @@ -215,6 +214,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { std::shared_ptr gcs_pub_sub_; std::shared_ptr gcs_table_storage_; std::shared_ptr redis_client_; + std::shared_ptr raylet_client_pool_; }; TEST_F(GcsPlacementGroupSchedulerTest, TestSpreadScheduleFailedWithZeroNode) { @@ -572,14 +572,16 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::SUCCESS); } -TEST_F(GcsPlacementGroupSchedulerTest, TestRescheduleWhenNodeDead) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); - AddNode(node0); - AddNode(node1); - ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); +TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { + int node_count = 3; + for (int index = 0; index < node_count; ++index) { + auto node = Mocker::GenNodeInfo(index); + AddNode(node); + } + ASSERT_EQ(3, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest( + "pg1", rpc::PlacementStrategy::STRICT_SPREAD); auto placement_group = std::make_shared(create_placement_group_request); @@ -594,38 +596,56 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestRescheduleWhenNodeDead) { }; scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); - ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); - ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); - WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); - WaitPendingDone(raylet_clients_[1]->commit_callbacks, 1); - ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); - ASSERT_TRUE(raylet_clients_[1]->GrantCommitBundleResources()); + + // Prepare bundle resources. + for (int index = 0; index < node_count; ++index) { + raylet_clients_[index]->GrantPrepareBundleResources(); + } + auto condition = [this]() { + absl::MutexLock lock(&placement_group_requests_mutex_); + return (int)(raylet_clients_[0]->commit_callbacks.size() + + raylet_clients_[1]->commit_callbacks.size() + + raylet_clients_[2]->commit_callbacks.size()) == 2; + }; + EXPECT_TRUE(WaitForCondition(condition, timeout_ms_.count())); + + // Filter out the nodes not scheduled by this placement group. + int node_index_not_scheduled = -1; + for (int index = 0; index < node_count; ++index) { + if (raylet_clients_[index]->commit_callbacks.empty()) { + node_index_not_scheduled = index; + break; + } + } + RAY_CHECK(node_index_not_scheduled != -1); + + // Commit bundle resources. + for (int index = 0; index < node_count; ++index) { + raylet_clients_[index]->GrantCommitBundleResources(); + } WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::SUCCESS); - auto bundles_on_node0 = - scheduler_->GetBundlesOnNode(NodeID::FromBinary(node0->node_id())); - ASSERT_EQ(1, bundles_on_node0.size()); - auto bundles_on_node1 = - scheduler_->GetBundlesOnNode(NodeID::FromBinary(node1->node_id())); - ASSERT_EQ(1, bundles_on_node1.size()); // One node is dead, reschedule the placement group. auto bundle_on_dead_node = placement_group->GetMutableBundle(0); bundle_on_dead_node->clear_node_id(); scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); - // TODO(ffbin): We need to see which node the other bundles that have been placed are - // deployed on, and spread them as far as possible. It will be implemented in the next - // pr. - auto commit_ready = [this]() { + // Prepare bundle resources. + for (int index = 0; index < node_count; ++index) { + raylet_clients_[index]->GrantPrepareBundleResources(); + } + + // Check the placement group scheduling results. + auto commit_ready = [this, node_index_not_scheduled]() { absl::MutexLock lock(&placement_group_requests_mutex_); - return raylet_clients_[0]->commit_callbacks.size() == 1 || - raylet_clients_[1]->commit_callbacks.size() == 1; + return raylet_clients_[node_index_not_scheduled]->commit_callbacks.size() == 1; }; - raylet_clients_[0]->GrantPrepareBundleResources(); - raylet_clients_[1]->GrantPrepareBundleResources(); EXPECT_TRUE(WaitForCondition(commit_ready, timeout_ms_.count())); - raylet_clients_[0]->GrantCommitBundleResources(); - raylet_clients_[1]->GrantCommitBundleResources(); + + // Commit bundle resources. + for (int index = 0; index < node_count; ++index) { + raylet_clients_[index]->GrantCommitBundleResources(); + } WaitPlacementGroupPendingDone(2, GcsPlacementGroupStatus::SUCCESS); } @@ -890,7 +910,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringRescheduling) { auto bundles_on_node1 = scheduler_->GetBundlesOnNode(NodeID::FromBinary(node1->node_id())); ASSERT_EQ(1, bundles_on_node1.size()); - // all nodes are dead, reschedule the placement group. + // All nodes are dead, reschedule the placement group. placement_group->GetMutableBundle(0)->clear_node_id(); placement_group->GetMutableBundle(1)->clear_node_id(); scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); @@ -944,7 +964,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommit) auto bundles_on_node1 = scheduler_->GetBundlesOnNode(NodeID::FromBinary(node1->node_id())); ASSERT_EQ(1, bundles_on_node1.size()); - // all nodes are dead, reschedule the placement group. + // All nodes are dead, reschedule the placement group. placement_group->GetMutableBundle(0)->clear_node_id(); placement_group->GetMutableBundle(1)->clear_node_id(); scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); diff --git a/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc index 5e3343d3e..9f732e0dd 100644 --- a/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc @@ -25,10 +25,10 @@ using ::testing::_; class GcsResourceManagerTest : public ::testing::Test { public: GcsResourceManagerTest() { - gcs_resource_manager_ = std::make_shared(); + gcs_resource_manager_ = std::make_shared(nullptr, nullptr); } - std::shared_ptr gcs_resource_manager_; + std::shared_ptr gcs_resource_manager_; }; TEST_F(GcsResourceManagerTest, TestBasic) { @@ -38,7 +38,7 @@ TEST_F(GcsResourceManagerTest, TestBasic) { std::unordered_map resource_map; resource_map[cpu_resource] = 10; ResourceSet resource_set(resource_map); - gcs_resource_manager_->UpdateResources(node_id, resource_set); + gcs_resource_manager_->UpdateResourceCapacity(node_id, resource_map); // Get and check cluster resources. const auto &cluster_resource = gcs_resource_manager_->GetClusterResources(); diff --git a/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc b/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc index c2f4657e0..fd2084168 100644 --- a/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc @@ -490,6 +490,46 @@ TEST_F(GcsServerTest, TestNodeInfo) { rpc::GcsNodeInfo_GcsNodeState::GcsNodeInfo_GcsNodeState_DEAD); } +TEST_F(GcsServerTest, TestHeartbeatWithNoRegistering) { + // Create gcs node info + auto gcs_node_info = Mocker::GenNodeInfo(); + + // Report heartbeat wit no registering + rpc::ReportHeartbeatRequest report_heartbeat_request; + report_heartbeat_request.mutable_heartbeat()->set_node_id(gcs_node_info->node_id()); + std::promise disconnected; + client_->ReportHeartbeat( + report_heartbeat_request, + [&disconnected](const Status &status, const rpc::ReportHeartbeatReply &reply) { + if (status.IsDisconnected()) { + disconnected.set_value(true); + } + }); + WaitReady(disconnected.get_future(), timeout_ms_); + + // Register node info + rpc::RegisterNodeRequest register_node_info_request; + register_node_info_request.mutable_node_info()->CopyFrom(*gcs_node_info); + ASSERT_TRUE(RegisterNode(register_node_info_request)); + std::vector node_info_list = GetAllNodeInfo(); + ASSERT_TRUE(node_info_list.size() == 1); + ASSERT_TRUE(node_info_list[0].state() == + rpc::GcsNodeInfo_GcsNodeState::GcsNodeInfo_GcsNodeState_ALIVE); + + // Report heartbeat + report_heartbeat_request.mutable_heartbeat()->set_node_id(gcs_node_info->node_id()); + ASSERT_TRUE(ReportHeartbeat(report_heartbeat_request)); + + // Unregister node info + rpc::UnregisterNodeRequest unregister_node_info_request; + unregister_node_info_request.set_node_id(gcs_node_info->node_id()); + ASSERT_TRUE(UnregisterNode(unregister_node_info_request)); + node_info_list = GetAllNodeInfo(); + ASSERT_TRUE(node_info_list.size() == 1); + ASSERT_TRUE(node_info_list[0].state() == + rpc::GcsNodeInfo_GcsNodeState::GcsNodeInfo_GcsNodeState_DEAD); +} + TEST_F(GcsServerTest, TestObjectInfo) { // Create object table data ObjectID object_id = ObjectID::FromRandom(); diff --git a/src/ray/gcs/gcs_server/test/gcs_server_test_util.h b/src/ray/gcs/gcs_server/test/gcs_server_test_util.h index 4477a1ef5..093b7c462 100644 --- a/src/ray/gcs/gcs_server/test/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/test/gcs_server_test_util.h @@ -56,8 +56,9 @@ struct GcsServerMocker { std::list> callbacks; }; - class MockRayletClient : public WorkerLeaseInterface { + class MockRayletClient : public RayletClientInterface { public: + /// WorkerLeaseInterface ray::Status ReturnWorker(int worker_port, const WorkerID &worker_id, bool disconnect_worker) override { if (disconnect_worker) { @@ -68,6 +69,7 @@ struct GcsServerMocker { return Status::OK(); } + /// WorkerLeaseInterface void RequestWorkerLease( const ray::TaskSpecification &resource_spec, const rpc::ClientCallback &callback, @@ -76,6 +78,7 @@ struct GcsServerMocker { callbacks.push_back(callback); } + /// WorkerLeaseInterface void ReleaseUnusedWorkers( const std::vector &workers_in_use, const rpc::ClientCallback &callback) override { @@ -83,6 +86,7 @@ struct GcsServerMocker { release_callbacks.push_back(callback); } + /// WorkerLeaseInterface void CancelWorkerLease( const TaskID &task_id, const rpc::ClientCallback &callback) override { @@ -145,21 +149,7 @@ struct GcsServerMocker { } } - ~MockRayletClient() {} - - int num_workers_requested = 0; - int num_workers_returned = 0; - int num_workers_disconnected = 0; - int num_leases_canceled = 0; - int num_release_unused_workers = 0; - NodeID node_id = NodeID::FromRandom(); - std::list> callbacks = {}; - std::list> cancel_callbacks = {}; - std::list> release_callbacks = {}; - }; - - class MockRayletResourceClient : public ResourceReserveInterface { - public: + /// ResourceReserveInterface void PrepareBundleResources( const BundleSpecification &bundle_spec, const ray::rpc::ClientCallback &callback) @@ -168,6 +158,7 @@ struct GcsServerMocker { lease_callbacks.push_back(callback); } + /// ResourceReserveInterface void CommitBundleResources( const BundleSpecification &bundle_spec, const ray::rpc::ClientCallback &callback) @@ -176,6 +167,7 @@ struct GcsServerMocker { commit_callbacks.push_back(callback); } + /// ResourceReserveInterface void CancelResourceReserve( BundleSpecification &bundle_spec, const ray::rpc::ClientCallback &callback) @@ -233,25 +225,42 @@ struct GcsServerMocker { } } - ~MockRayletResourceClient() {} + /// PinObjectsInterface + void PinObjectIDs( + const rpc::Address &caller_address, const std::vector &object_ids, + const ray::rpc::ClientCallback &callback) override {} + /// DependencyWaiterInterface + ray::Status WaitForDirectActorCallArgs( + const std::vector &references, int64_t tag) override { + return ray::Status::OK(); + } + + ~MockRayletClient() {} + + int num_workers_requested = 0; + int num_workers_returned = 0; + int num_workers_disconnected = 0; + int num_leases_canceled = 0; + int num_release_unused_workers = 0; + NodeID node_id = NodeID::FromRandom(); + std::list> callbacks = {}; + std::list> cancel_callbacks = {}; + std::list> release_callbacks = {}; int num_lease_requested = 0; int num_return_requested = 0; int num_commit_requested = 0; + int num_release_unused_bundles_requested = 0; - NodeID node_id = NodeID::FromRandom(); std::list> lease_callbacks = {}; std::list> commit_callbacks = {}; std::list> return_callbacks = {}; }; + class MockedGcsActorScheduler : public gcs::GcsActorScheduler { public: using gcs::GcsActorScheduler::GcsActorScheduler; - void ResetLeaseClientFactory(gcs::LeaseClientFactoryFn lease_client_factory) { - lease_client_factory_ = std::move(lease_client_factory); - } - void TryLeaseWorkerFromNodeAgain(std::shared_ptr actor, std::shared_ptr node) { DoRetryLeasingWorkerFromNode(std::move(actor), std::move(node)); @@ -280,11 +289,6 @@ struct GcsServerMocker { class MockedGcsPlacementGroupScheduler : public gcs::GcsPlacementGroupScheduler { public: using gcs::GcsPlacementGroupScheduler::GcsPlacementGroupScheduler; - - void ResetLeaseClientFactory( - gcs::ReserveResourceClientFactoryFn lease_client_factory) { - lease_client_factory_ = std::move(lease_client_factory); - } }; class MockedGcsActorTable : public gcs::GcsActorTable { public: @@ -362,36 +366,13 @@ struct GcsServerMocker { bool IsRemoved(const NodeID &node_id) const override { return false; } - Status AsyncGetResources( - const NodeID &node_id, - const gcs::OptionalItemCallback &callback) override { - return Status::NotImplemented(""); - } - - Status AsyncUpdateResources(const NodeID &node_id, const ResourceMap &resources, - const gcs::StatusCallback &callback) override { - return Status::NotImplemented(""); - } - - Status AsyncDeleteResources(const NodeID &node_id, - const std::vector &resource_names, - const gcs::StatusCallback &callback) override { - return Status::NotImplemented(""); - } - - Status AsyncSubscribeToResources( - const gcs::ItemCallback &subscribe, - const gcs::StatusCallback &done) override { - return Status::NotImplemented(""); - } - Status AsyncReportHeartbeat(const std::shared_ptr &data_ptr, const gcs::StatusCallback &callback) override { return Status::NotImplemented(""); } - Status AsyncSubscribeBatchHeartbeat( - const gcs::ItemCallback &subscribe, + Status AsyncSubscribeBatchedResourceUsage( + const gcs::ItemCallback &subscribe, const gcs::StatusCallback &done) override { return Status::NotImplemented(""); } diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h index 19a0923ac..06748dffb 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ b/src/ray/gcs/pubsub/gcs_pub_sub.h @@ -32,7 +32,7 @@ namespace gcs { #define OBJECT_CHANNEL "OBJECT" #define TASK_CHANNEL "TASK" #define TASK_LEASE_CHANNEL "TASK_LEASE" -#define HEARTBEAT_BATCH_CHANNEL "HEARTBEAT_BATCH" +#define RESOURCES_BATCH_CHANNEL "RESOURCES_BATCH" #define ERROR_INFO_CHANNEL "ERROR_INFO" /// \class GcsPubSub diff --git a/src/ray/gcs/redis_accessor.cc b/src/ray/gcs/redis_accessor.cc index b048e32ad..bd3fe0604 100644 --- a/src/ray/gcs/redis_accessor.cc +++ b/src/ray/gcs/redis_accessor.cc @@ -421,8 +421,7 @@ Status RedisObjectInfoAccessor::AsyncUnsubscribeToLocations(const ObjectID &obje RedisNodeInfoAccessor::RedisNodeInfoAccessor(RedisGcsClient *client_impl) : client_impl_(client_impl), - resource_sub_executor_(client_impl_->resource_table()), - heartbeat_batch_sub_executor_(client_impl->heartbeat_batch_table()) {} + resource_usage_batch_sub_executor_(client_impl->resource_usage_batch_table()) {} Status RedisNodeInfoAccessor::RegisterSelf(const GcsNodeInfo &local_node_info, const StatusCallback &callback) { @@ -530,21 +529,29 @@ Status RedisNodeInfoAccessor::AsyncReportHeartbeat( return heartbeat_table.Add(JobID::Nil(), node_id, data_ptr, on_done); } -void RedisNodeInfoAccessor::AsyncReReportHeartbeat() {} +Status RedisNodeInfoAccessor::AsyncReportResourceUsage( + const std::shared_ptr &data_ptr, const StatusCallback &callback) { + return Status::Invalid("Not implemented"); +} -Status RedisNodeInfoAccessor::AsyncSubscribeBatchHeartbeat( - const ItemCallback &subscribe, const StatusCallback &done) { +void RedisNodeInfoAccessor::AsyncReReportResourceUsage() {} + +Status RedisNodeInfoAccessor::AsyncSubscribeBatchedResourceUsage( + const ItemCallback &subscribe, const StatusCallback &done) { RAY_CHECK(subscribe != nullptr); auto on_subscribe = [subscribe](const NodeID &node_id, - const HeartbeatBatchTableData &data) { + const ResourceUsageBatchData &data) { subscribe(data); }; - return heartbeat_batch_sub_executor_.AsyncSubscribeAll(NodeID::Nil(), on_subscribe, - done); + return resource_usage_batch_sub_executor_.AsyncSubscribeAll(NodeID::Nil(), on_subscribe, + done); } -Status RedisNodeInfoAccessor::AsyncGetResources( +RedisNodeResourceInfoAccessor::RedisNodeResourceInfoAccessor(RedisGcsClient *client_impl) + : client_impl_(client_impl), resource_sub_executor_(client_impl_->resource_table()) {} + +Status RedisNodeResourceInfoAccessor::AsyncGetResources( const NodeID &node_id, const OptionalItemCallback &callback) { RAY_CHECK(callback != nullptr); auto on_done = [callback](RedisGcsClient *client, const NodeID &id, @@ -560,9 +567,8 @@ Status RedisNodeInfoAccessor::AsyncGetResources( return resource_table.Lookup(JobID::Nil(), node_id, on_done); } -Status RedisNodeInfoAccessor::AsyncUpdateResources(const NodeID &node_id, - const ResourceMap &resources, - const StatusCallback &callback) { +Status RedisNodeResourceInfoAccessor::AsyncUpdateResources( + const NodeID &node_id, const ResourceMap &resources, const StatusCallback &callback) { Hash::HashCallback on_done = nullptr; if (callback != nullptr) { on_done = [callback](RedisGcsClient *client, const NodeID &node_id, @@ -573,7 +579,7 @@ Status RedisNodeInfoAccessor::AsyncUpdateResources(const NodeID &node_id, return resource_table.Update(JobID::Nil(), node_id, resources, on_done); } -Status RedisNodeInfoAccessor::AsyncDeleteResources( +Status RedisNodeResourceInfoAccessor::AsyncDeleteResources( const NodeID &node_id, const std::vector &resource_names, const StatusCallback &callback) { Hash::HashRemoveCallback on_done = nullptr; @@ -588,7 +594,7 @@ Status RedisNodeInfoAccessor::AsyncDeleteResources( return resource_table.RemoveEntries(JobID::Nil(), node_id, resource_names, on_done); } -Status RedisNodeInfoAccessor::AsyncSubscribeToResources( +Status RedisNodeResourceInfoAccessor::AsyncSubscribeToResources( const ItemCallback &subscribe, const StatusCallback &done) { RAY_CHECK(subscribe != nullptr); auto on_subscribe = [subscribe](const NodeID &id, @@ -694,6 +700,11 @@ Status RedisPlacementGroupInfoAccessor::AsyncGetAll( return Status::Invalid("Not implemented"); } +Status RedisPlacementGroupInfoAccessor::AsyncWaitUntilReady( + const PlacementGroupID &placement_group_id, const StatusCallback &callback) { + return Status::Invalid("Not implemented"); +} + } // namespace gcs } // namespace ray diff --git a/src/ray/gcs/redis_accessor.h b/src/ray/gcs/redis_accessor.h index baa63514b..c8263d0c8 100644 --- a/src/ray/gcs/redis_accessor.h +++ b/src/ray/gcs/redis_accessor.h @@ -326,36 +326,21 @@ class RedisNodeInfoAccessor : public NodeInfoAccessor { bool IsRemoved(const NodeID &node_id) const override; - Status AsyncGetResources(const NodeID &node_id, - const OptionalItemCallback &callback) override; - - Status AsyncGetAllAvailableResources( - const MultiItemCallback &callback) override { - return Status::NotImplemented("AsyncGetAllAvailableResources not implemented"); - } - - Status AsyncUpdateResources(const NodeID &node_id, const ResourceMap &resources, - const StatusCallback &callback) override; - - Status AsyncDeleteResources(const NodeID &node_id, - const std::vector &resource_names, - const StatusCallback &callback) override; - - Status AsyncSubscribeToResources(const ItemCallback &subscribe, - const StatusCallback &done) override; - Status AsyncReportHeartbeat(const std::shared_ptr &data_ptr, const StatusCallback &callback) override; - void AsyncReReportHeartbeat() override; + Status AsyncReportResourceUsage(const std::shared_ptr &data_ptr, + const StatusCallback &callback) override; - Status AsyncGetAllHeartbeat( - const ItemCallback &callback) override { - return Status::NotImplemented("AsyncGetAllHeartbeat not implemented"); + void AsyncReReportResourceUsage() override; + + Status AsyncGetAllResourceUsage( + const ItemCallback &callback) override { + return Status::NotImplemented("AsyncGetAllResourceUsage not implemented"); } - Status AsyncSubscribeBatchHeartbeat( - const ItemCallback &subscribe, + Status AsyncSubscribeBatchedResourceUsage( + const ItemCallback &subscribe, const StatusCallback &done) override; void AsyncResubscribe(bool is_pubsub_server_restarted) override {} @@ -371,16 +356,49 @@ class RedisNodeInfoAccessor : public NodeInfoAccessor { return Status::NotImplemented("GetInternalConfig not implemented."); } + private: + RedisGcsClient *client_impl_{nullptr}; + + typedef SubscriptionExecutor + HeartbeatBatchSubscriptionExecutor; + HeartbeatBatchSubscriptionExecutor resource_usage_batch_sub_executor_; +}; + +/// \class RedisNodeResourceInfoAccessor +/// RedisNodeResourceInfoAccessor is an implementation of `NodeResourceInfoAccessor` +/// that uses Redis as the backend storage. +class RedisNodeResourceInfoAccessor : public NodeResourceInfoAccessor { + public: + explicit RedisNodeResourceInfoAccessor(RedisGcsClient *client_impl); + + virtual ~RedisNodeResourceInfoAccessor() {} + + Status AsyncGetResources(const NodeID &node_id, + const OptionalItemCallback &callback) override; + + Status AsyncGetAllAvailableResources( + const MultiItemCallback &callback) override { + return Status::NotImplemented("AsyncGetAllAvailableResources not implemented"); + } + + Status AsyncUpdateResources(const NodeID &node_id, const ResourceMap &resources, + const StatusCallback &callback) override; + + Status AsyncDeleteResources(const NodeID &node_id, + const std::vector &resource_names, + const StatusCallback &callback) override; + + Status AsyncSubscribeToResources(const ItemCallback &subscribe, + const StatusCallback &done) override; + + void AsyncResubscribe(bool is_pubsub_server_restarted) override {} + private: RedisGcsClient *client_impl_{nullptr}; typedef SubscriptionExecutor DynamicResourceSubscriptionExecutor; DynamicResourceSubscriptionExecutor resource_sub_executor_; - - typedef SubscriptionExecutor - HeartbeatBatchSubscriptionExecutor; - HeartbeatBatchSubscriptionExecutor heartbeat_batch_sub_executor_; }; /// \class RedisErrorInfoAccessor @@ -466,6 +484,9 @@ class RedisPlacementGroupInfoAccessor : public PlacementGroupInfoAccessor { Status AsyncGetAll( const MultiItemCallback &callback) override; + + Status AsyncWaitUntilReady(const PlacementGroupID &placement_group_id, + const StatusCallback &callback) override; }; } // namespace gcs diff --git a/src/ray/gcs/redis_context.cc b/src/ray/gcs/redis_context.cc index 9034eaf9c..c9f32c711 100644 --- a/src/ray/gcs/redis_context.cc +++ b/src/ray/gcs/redis_context.cc @@ -304,7 +304,7 @@ Status ConnectWithoutRetries(const std::string &address, int port, oss << "Could not allocate Redis context."; } else if (newContext->err) { oss << "Could not establish connection to Redis " << address << ":" << port - << " (context.err = " << newContext->err << ")"; + << " (context.err = " << newContext->err << ")."; } return Status::RedisError(errorMessage); } diff --git a/src/ray/gcs/redis_gcs_client.cc b/src/ray/gcs/redis_gcs_client.cc index 35fd0506e..1b2359346 100644 --- a/src/ray/gcs/redis_gcs_client.cc +++ b/src/ray/gcs/redis_gcs_client.cc @@ -55,7 +55,7 @@ Status RedisGcsClient::Connect(boost::asio::io_service &io_service) { node_table_.reset(new NodeTable({primary_context}, this)); job_table_.reset(new JobTable({primary_context}, this)); - heartbeat_batch_table_.reset(new HeartbeatBatchTable({primary_context}, this)); + resource_usage_batch_table_.reset(new ResourceUsageBatchTable({primary_context}, this)); // Tables below would be sharded. object_table_.reset(new ObjectTable(shard_contexts, this)); raylet_task_table_.reset(new raylet::TaskTable(shard_contexts, this, command_type_)); @@ -71,6 +71,7 @@ Status RedisGcsClient::Connect(boost::asio::io_service &io_service) { job_accessor_.reset(new RedisJobInfoAccessor(this)); object_accessor_.reset(new RedisObjectInfoAccessor(this)); node_accessor_.reset(new RedisNodeInfoAccessor(this)); + node_resource_accessor_.reset(new RedisNodeResourceInfoAccessor(this)); task_accessor_.reset(new RedisTaskInfoAccessor(this)); error_accessor_.reset(new RedisErrorInfoAccessor(this)); stats_accessor_.reset(new RedisStatsInfoAccessor(this)); @@ -128,8 +129,8 @@ NodeTable &RedisGcsClient::node_table() { return *node_table_; } HeartbeatTable &RedisGcsClient::heartbeat_table() { return *heartbeat_table_; } -HeartbeatBatchTable &RedisGcsClient::heartbeat_batch_table() { - return *heartbeat_batch_table_; +ResourceUsageBatchTable &RedisGcsClient::resource_usage_batch_table() { + return *resource_usage_batch_table_; } JobTable &RedisGcsClient::job_table() { return *job_table_; } diff --git a/src/ray/gcs/redis_gcs_client.h b/src/ray/gcs/redis_gcs_client.h index 7d14826ba..748b1da72 100644 --- a/src/ray/gcs/redis_gcs_client.h +++ b/src/ray/gcs/redis_gcs_client.h @@ -93,7 +93,7 @@ class RAY_EXPORT RedisGcsClient : public GcsClient { /// Implements the Nodes() interface. NodeTable &node_table(); HeartbeatTable &heartbeat_table(); - HeartbeatBatchTable &heartbeat_batch_table(); + ResourceUsageBatchTable &resource_usage_batch_table(); DynamicResourceTable &resource_table(); /// Implements the Tasks() interface. virtual raylet::TaskTable &raylet_task_table(); @@ -118,7 +118,7 @@ class RAY_EXPORT RedisGcsClient : public GcsClient { std::unique_ptr task_reconstruction_log_; std::unique_ptr task_lease_table_; std::unique_ptr heartbeat_table_; - std::unique_ptr heartbeat_batch_table_; + std::unique_ptr resource_usage_batch_table_; std::unique_ptr profile_table_; std::unique_ptr node_table_; std::unique_ptr resource_table_; diff --git a/src/ray/gcs/subscription_executor.cc b/src/ray/gcs/subscription_executor.cc index 4d90d3aac..d9617985a 100644 --- a/src/ray/gcs/subscription_executor.cc +++ b/src/ray/gcs/subscription_executor.cc @@ -206,7 +206,8 @@ template class SubscriptionExecutor, TaskLeaseTable>; template class SubscriptionExecutor; -template class SubscriptionExecutor; +template class SubscriptionExecutor; template class SubscriptionExecutor; } // namespace gcs diff --git a/src/ray/gcs/tables.cc b/src/ray/gcs/tables.cc index 17ad126fd..2017d05de 100644 --- a/src/ray/gcs/tables.cc +++ b/src/ray/gcs/tables.cc @@ -829,12 +829,12 @@ template class Log; template class Log; template class Table; template class Table; -template class Table; +template class Table; template class Log; template class Log; template class Log; template class Log; -template class Log; +template class Log; template class Log; template class Table; template class Table; diff --git a/src/ray/gcs/tables.h b/src/ray/gcs/tables.h index 11683e6f4..c7c647162 100644 --- a/src/ray/gcs/tables.h +++ b/src/ray/gcs/tables.h @@ -39,12 +39,12 @@ using rpc::ErrorTableData; using rpc::GcsChangeMode; using rpc::GcsEntry; using rpc::GcsNodeInfo; -using rpc::HeartbeatBatchTableData; using rpc::HeartbeatTableData; using rpc::JobTableData; using rpc::ObjectTableData; using rpc::ProfileTableData; using rpc::ResourceTableData; +using rpc::ResourceUsageBatchData; using rpc::TablePrefix; using rpc::TablePubsub; using rpc::TaskLeaseData; @@ -688,15 +688,15 @@ class HeartbeatTable : public Table { virtual ~HeartbeatTable() {} }; -class HeartbeatBatchTable : public Table { +class ResourceUsageBatchTable : public Table { public: - HeartbeatBatchTable(const std::vector> &contexts, - RedisGcsClient *client) + ResourceUsageBatchTable(const std::vector> &contexts, + RedisGcsClient *client) : Table(contexts, client) { - pubsub_channel_ = TablePubsub::HEARTBEAT_BATCH_PUBSUB; - prefix_ = TablePrefix::HEARTBEAT_BATCH; + pubsub_channel_ = TablePubsub::RESOURCE_USAGE_BATCH_PUBSUB; + prefix_ = TablePrefix::RESOURCE_USAGE_BATCH; } - virtual ~HeartbeatBatchTable() {} + virtual ~ResourceUsageBatchTable() {} }; class JobTable : public Log { diff --git a/src/ray/gcs/test/gcs_test_util.h b/src/ray/gcs/test/gcs_test_util.h index 40c478c37..bf908c3a2 100644 --- a/src/ray/gcs/test/gcs_test_util.h +++ b/src/ray/gcs/test/gcs_test_util.h @@ -18,6 +18,7 @@ #include #include "gmock/gmock.h" +#include "ray/common/bundle_spec.h" #include "ray/common/placement_group.h" #include "ray/common/task/task.h" #include "ray/common/task/task_util.h" @@ -41,8 +42,9 @@ struct Mocker { builder.SetCommonTaskSpec(task_id, name + ":" + empty_descriptor->CallString(), Language::PYTHON, empty_descriptor, job_id, TaskID::Nil(), 0, TaskID::Nil(), owner_address, 1, resource, resource, - std::make_pair(PlacementGroupID::Nil(), -1), true); - builder.SetActorCreationTaskSpec(actor_id, max_restarts, {}, 1, detached, name); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); + builder.SetActorCreationTaskSpec(actor_id, max_restarts, /*max_task_retries=*/0, {}, + 1, detached, name); return builder.Build(); } @@ -78,6 +80,20 @@ struct Mocker { return request; } + static BundleSpecification GenBundleCreation( + const PlacementGroupID &placement_group_id, const int bundle_index, + std::unordered_map &unit_resource) { + rpc::Bundle bundle; + auto mutable_bundle_id = bundle.mutable_bundle_id(); + mutable_bundle_id->set_bundle_index(bundle_index); + mutable_bundle_id->set_placement_group_id(placement_group_id.Binary()); + auto mutable_unit_resources = bundle.mutable_unit_resources(); + for (auto &resource : unit_resource) { + mutable_unit_resources->insert({resource.first, resource.second}); + } + return BundleSpecification(bundle); + } + static PlacementGroupSpecification GenPlacementGroupCreation( const std::string &name, std::vector> &bundles, diff --git a/src/ray/gcs/test/redis_node_info_accessor_test.cc b/src/ray/gcs/test/redis_node_info_accessor_test.cc index 49b31b09f..e4435184e 100644 --- a/src/ray/gcs/test/redis_node_info_accessor_test.cc +++ b/src/ray/gcs/test/redis_node_info_accessor_test.cc @@ -25,7 +25,7 @@ namespace gcs { class NodeDynamicResourceTest : public AccessorTestBase { protected: - typedef NodeInfoAccessor::ResourceMap ResourceMap; + typedef NodeResourceInfoAccessor::ResourceMap ResourceMap; virtual void GenTestData() { for (size_t node_index = 0; node_index < node_number_; ++node_index) { NodeID id = NodeID::FromRandom(); @@ -56,13 +56,14 @@ class NodeDynamicResourceTest : public AccessorTestBaseNodes(); + NodeResourceInfoAccessor &node_resource_accessor = gcs_client_->NodeResources(); for (const auto &node_rs : id_to_resource_map_) { ++pending_count_; const NodeID &id = node_rs.first; // Update - Status status = node_accessor.AsyncUpdateResources( - node_rs.first, node_rs.second, [this, &node_accessor, id](Status status) { + Status status = node_resource_accessor.AsyncUpdateResources( + node_rs.first, node_rs.second, + [this, &node_resource_accessor, id](Status status) { RAY_CHECK_OK(status); auto get_callback = [this, id](Status status, const boost::optional &result) { @@ -73,7 +74,7 @@ TEST_F(NodeDynamicResourceTest, UpdateAndGet) { ASSERT_EQ(it->second.size(), result->size()); }; // Get - status = node_accessor.AsyncGetResources(id, get_callback); + status = node_resource_accessor.AsyncGetResources(id, get_callback); RAY_CHECK_OK(status); }); } @@ -81,15 +82,15 @@ TEST_F(NodeDynamicResourceTest, UpdateAndGet) { } TEST_F(NodeDynamicResourceTest, Delete) { - NodeInfoAccessor &node_accessor = gcs_client_->Nodes(); + NodeResourceInfoAccessor &node_resource_accessor = gcs_client_->NodeResources(); for (const auto &node_rs : id_to_resource_map_) { ++pending_count_; // Update - Status status = node_accessor.AsyncUpdateResources(node_rs.first, node_rs.second, - [this](Status status) { - RAY_CHECK_OK(status); - --pending_count_; - }); + Status status = node_resource_accessor.AsyncUpdateResources( + node_rs.first, node_rs.second, [this](Status status) { + RAY_CHECK_OK(status); + --pending_count_; + }); } WaitPendingDone(wait_pending_timeout_); @@ -97,11 +98,11 @@ TEST_F(NodeDynamicResourceTest, Delete) { ++pending_count_; const NodeID &id = node_rs.first; // Delete - Status status = node_accessor.AsyncDeleteResources( - id, resource_to_delete_, [this, &node_accessor, id](Status status) { + Status status = node_resource_accessor.AsyncDeleteResources( + id, resource_to_delete_, [this, &node_resource_accessor, id](Status status) { RAY_CHECK_OK(status); // Get - status = node_accessor.AsyncGetResources( + status = node_resource_accessor.AsyncGetResources( id, [this, id](Status status, const boost::optional &result) { --pending_count_; RAY_CHECK_OK(status); @@ -115,15 +116,15 @@ TEST_F(NodeDynamicResourceTest, Delete) { } TEST_F(NodeDynamicResourceTest, Subscribe) { - NodeInfoAccessor &node_accessor = gcs_client_->Nodes(); + NodeResourceInfoAccessor &node_resource_accessor = gcs_client_->NodeResources(); for (const auto &node_rs : id_to_resource_map_) { ++pending_count_; // Update - Status status = node_accessor.AsyncUpdateResources(node_rs.first, node_rs.second, - [this](Status status) { - RAY_CHECK_OK(status); - --pending_count_; - }); + Status status = node_resource_accessor.AsyncUpdateResources( + node_rs.first, node_rs.second, [this](Status status) { + RAY_CHECK_OK(status); + --pending_count_; + }); } WaitPendingDone(wait_pending_timeout_); @@ -147,18 +148,18 @@ TEST_F(NodeDynamicResourceTest, Subscribe) { // Subscribe ++pending_count_; - Status status = node_accessor.AsyncSubscribeToResources(subscribe, done); + Status status = node_resource_accessor.AsyncSubscribeToResources(subscribe, done); RAY_CHECK_OK(status); for (const auto &node_rs : id_to_resource_map_) { // Delete ++pending_count_; ++sub_pending_count_; - Status status = node_accessor.AsyncDeleteResources(node_rs.first, resource_to_delete_, - [this](Status status) { - RAY_CHECK_OK(status); - --pending_count_; - }); + Status status = node_resource_accessor.AsyncDeleteResources( + node_rs.first, resource_to_delete_, [this](Status status) { + RAY_CHECK_OK(status); + --pending_count_; + }); RAY_CHECK_OK(status); } diff --git a/src/ray/object_manager/common.h b/src/ray/object_manager/common.h index eacfb4b13..c7c531ffc 100644 --- a/src/ray/object_manager/common.h +++ b/src/ray/object_manager/common.h @@ -1,8 +1,12 @@ #pragma once +#include #include +#include "ray/common/id.h" +#include "ray/object_manager/format/object_manager_generated.h" + namespace ray { /// A callback to asynchronously spill objects when space is needed. @@ -18,4 +22,8 @@ using SpillObjectsCallback = /// A callback to call when space has been released. using SpaceReleasedCallback = std::function; +/// A callback to call when a spilled object needs to be returned to the object store. +using RestoreSpilledObjectCallback = std::function)>; + } // namespace ray diff --git a/src/ray/object_manager/object_directory.h b/src/ray/object_manager/object_directory.h index 0396309bd..7133d1e94 100644 --- a/src/ray/object_manager/object_directory.h +++ b/src/ray/object_manager/object_directory.h @@ -40,6 +40,11 @@ struct RemoteConnectionInfo { uint16_t port; }; +/// Callback for object location notifications. +using OnLocationsFound = + std::function &, const std::string &)>; + class ObjectDirectoryInterface { public: virtual ~ObjectDirectoryInterface() {} @@ -58,12 +63,7 @@ class ObjectDirectoryInterface { /// \return A vector of information for all connected remote object managers. virtual std::vector LookupAllRemoteConnections() const = 0; - /// Callback for object location notifications. - using OnLocationsFound = - std::function &, const std::string &)>; - - /// Lookup object locations. Callback may be invoked with empty list of node ids. + /// Lookup object locations. Callback may be invoked with empty list of client ids. /// /// \param object_id The object's ObjectID. /// \param callback Invoked with (possibly empty) list of node ids and object_id. diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 90380e1b0..2eb641d04 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -57,25 +57,47 @@ ObjectManager::ObjectManager(asio::io_service &main_service, const NodeID &self_ RestoreSpilledObjectCallback restore_spilled_object, SpillObjectsCallback spill_objects_callback, std::function object_store_full_callback) - : self_node_id_(self_node_id), + : main_service_(&main_service), + self_node_id_(self_node_id), config_(config), object_directory_(std::move(object_directory)), object_store_internal_(config, spill_objects_callback, object_store_full_callback), buffer_pool_(config_.store_socket_name, config_.object_chunk_size), rpc_work_(rpc_service_), - gen_(std::chrono::high_resolution_clock::now().time_since_epoch().count()), object_manager_server_("ObjectManager", config_.object_manager_port, config_.rpc_service_threads_number), object_manager_service_(rpc_service_, *this), client_call_manager_(main_service, config_.rpc_service_threads_number), - restore_spilled_object_(restore_spilled_object) { + restore_spilled_object_(restore_spilled_object), + pull_retry_timer_(*main_service_, + boost::posix_time::milliseconds(config.timer_freq_ms)) { RAY_CHECK(config_.rpc_service_threads_number > 0); - main_service_ = &main_service; + + const auto &object_is_local = [this](const ObjectID &object_id) { + return local_objects_.count(object_id) != 0; + }; + const auto &send_pull_request = [this](const ObjectID &object_id, + const NodeID &client_id) { + SendPullRequest(object_id, client_id); + }; + const auto &get_time = []() { return absl::GetCurrentTimeNanos() / 1e9; }; + pull_manager_.reset(new PullManager(self_node_id_, object_is_local, send_pull_request, + restore_spilled_object_, get_time, + config.pull_timeout_ms)); push_manager_.reset(new PushManager(/* max_chunks_in_flight= */ std::max( static_cast(1L), static_cast(config_.max_bytes_in_flight / config_.object_chunk_size)))); + pull_retry_timer_.async_wait([this](const boost::system::error_code &e) { + RAY_CHECK(!e) << "The raylet's object manager has failed unexpectedly with error: " + << e + << ". Please file a bug report on here: " + "https://github.com/ray-project/ray/issues"; + + Tick(); + }); + if (plasma::plasma_store_runner) { store_notification_ = std::make_shared(main_service); plasma::plasma_store_runner->SetNotificationListener(store_notification_); @@ -178,171 +200,48 @@ ray::Status ObjectManager::SubscribeObjDeleted( ray::Status ObjectManager::Pull(const ObjectID &object_id, const rpc::Address &owner_address) { - RAY_LOG(DEBUG) << "Pull on " << self_node_id_ << " of object " << object_id; - // Check if object is already local. - if (local_objects_.count(object_id) != 0) { - RAY_LOG(ERROR) << object_id << " attempted to pull an object that's already local."; - return ray::Status::OK(); - } - if (pull_requests_.find(object_id) != pull_requests_.end()) { - RAY_LOG(DEBUG) << object_id << " has inflight pull_requests, skipping."; - return ray::Status::OK(); + if (!pull_manager_->Pull(object_id, owner_address)) { + // If we don't need to pull, the object is either already local or this is a duplicate + // request. + return Status::OK(); } - pull_requests_.emplace(object_id, PullRequest()); + const auto &callback = [this](const ObjectID &object_id, + const std::unordered_set &client_ids, + const std::string &spilled_url) { + pull_manager_->OnLocationChange(object_id, client_ids, spilled_url); + }; + // Subscribe to object notifications. A notification will be received every // time the set of node IDs for the object changes. Notifications will also // be received if the list of locations is empty. The set of node IDs has // no ordering guarantee between notifications. - return object_directory_->SubscribeObjectLocations( - object_directory_pull_callback_id_, object_id, owner_address, - [this](const ObjectID &object_id, const std::unordered_set &node_ids, - const std::string &spilled_url) { - // Exit if the Pull request has already been fulfilled or canceled. - auto it = pull_requests_.find(object_id); - if (it == pull_requests_.end()) { - return; - } - // Reset the list of nodes that are now expected to have the object. - // NOTE(swang): Since we are overwriting the previous list of nodes, - // we may end up sending a duplicate request to the same node as - // before. - it->second.node_locations = std::vector(node_ids.begin(), node_ids.end()); - if (!spilled_url.empty()) { - // Try to restore the spilled object. - restore_spilled_object_(object_id, spilled_url, - [this, object_id](const ray::Status &status) { - // Fall back to fetching from another object manager. - if (!status.ok()) { - TryPull(object_id); - } - }); - } else if (it->second.node_locations.empty()) { - // The object locations are now empty, so we should wait for the next - // notification about a new object location. Cancel the timer until - // the next Pull attempt since there are no more nodes to try. - if (it->second.retry_timer != nullptr) { - it->second.retry_timer->cancel(); - it->second.timer_set = false; - } - } else { - // New object locations were found, so begin trying to pull from a - // node. This will be called every time a new node location - // appears. - TryPull(object_id); - } - }); + return object_directory_->SubscribeObjectLocations(object_directory_pull_callback_id_, + object_id, owner_address, callback); } -void ObjectManager::TryPull(const ObjectID &object_id) { - auto it = pull_requests_.find(object_id); - if (it == pull_requests_.end()) { - return; - } - - auto &node_vector = it->second.node_locations; - - // The timer should never fire if there are no expected node locations. - if (node_vector.empty()) { - return; - } - - RAY_CHECK(local_objects_.count(object_id) == 0); - // Make sure that there is at least one node which is not the local node. - // TODO(rkn): It may actually be possible for this check to fail. - if (node_vector.size() == 1 && node_vector[0] == self_node_id_) { - RAY_LOG(WARNING) << "The object manager with ID " << self_node_id_ - << " is trying to pull object " << object_id - << " but the object table suggests that this object manager " - << "already has the object. The object may have been evicted. It is " - << "most likely due to memory pressure, object pull has been " - << "requested before object location is updated."; - it->second.timer_set = false; - return; - } - - // Choose a random node to pull the object from. - // Generate a random index. - std::uniform_int_distribution distribution(0, node_vector.size() - 1); - int node_index = distribution(gen_); - NodeID node_id = node_vector[node_index]; - // If the object manager somehow ended up choosing itself, choose a different - // object manager. - if (node_id == self_node_id_) { - std::swap(node_vector[node_index], node_vector[node_vector.size() - 1]); - node_vector.pop_back(); - RAY_LOG(WARNING) - << "The object manager with ID " << self_node_id_ << " is trying to pull object " - << object_id << " but the object table suggests that this object manager " - << "already has the object. It is most likely due to memory pressure, object " - << "pull has been requested before object location is updated."; - node_id = node_vector[node_index % node_vector.size()]; - RAY_CHECK(node_id != self_node_id_); - } - - RAY_LOG(DEBUG) << "Sending pull request from " << self_node_id_ << " to " << node_id - << " of object " << object_id; - - auto rpc_client = GetRpcClient(node_id); +void ObjectManager::SendPullRequest(const ObjectID &object_id, const NodeID &client_id) { + auto rpc_client = GetRpcClient(client_id); if (rpc_client) { - // Try pulling from the node. - rpc_service_.post([this, object_id, node_id, rpc_client]() { - SendPullRequest(object_id, node_id, rpc_client); + // Try pulling from the client. + rpc_service_.post([this, object_id, client_id, rpc_client]() { + rpc::PullRequest pull_request; + pull_request.set_object_id(object_id.Binary()); + pull_request.set_node_id(self_node_id_.Binary()); + + rpc_client->Pull(pull_request, [object_id, client_id](const Status &status, + const rpc::PullReply &reply) { + if (!status.ok()) { + RAY_LOG(WARNING) << "Send pull " << object_id << " request to client " + << client_id << " failed due to" << status.message(); + } + }); }); } else { RAY_LOG(ERROR) << "Couldn't send pull request from " << self_node_id_ << " to " - << node_id << " of object " << object_id + << client_id << " of object " << object_id << " , setup rpc connection failed."; } - - // If there are more nodes to try, try them in succession, with a timeout - // in between each try. - if (!it->second.node_locations.empty()) { - if (it->second.retry_timer == nullptr) { - // Set the timer if we haven't already. - it->second.retry_timer = std::unique_ptr( - new boost::asio::deadline_timer(*main_service_)); - } - - // Wait for a timeout. If we receive the object or a caller Cancels the - // Pull within the timeout, then nothing will happen. Otherwise, the timer - // will fire and the next node in the list will be tried. - boost::posix_time::milliseconds retry_timeout(config_.pull_timeout_ms); - it->second.retry_timer->expires_from_now(retry_timeout); - it->second.retry_timer->async_wait( - [this, object_id](const boost::system::error_code &error) { - if (!error) { - // Try the Pull from the next node. - TryPull(object_id); - } else { - // Check that the error was due to the timer being canceled. - RAY_CHECK(error == boost::asio::error::operation_aborted); - } - }); - // Record that we set the timer until the next attempt. - it->second.timer_set = true; - } else { - // The timer is not reset since there are no more nodes to try. Go back - // to waiting for more notifications. Once we receive a new object location - // from the object directory, then the Pull will be retried. - it->second.timer_set = false; - } -}; - -void ObjectManager::SendPullRequest( - const ObjectID &object_id, const NodeID &node_id, - std::shared_ptr rpc_client) { - rpc::PullRequest pull_request; - pull_request.set_object_id(object_id.Binary()); - pull_request.set_node_id(self_node_id_.Binary()); - - rpc_client->Pull(pull_request, [object_id, node_id](const Status &status, - const rpc::PullReply &reply) { - if (!status.ok()) { - RAY_LOG(WARNING) << "Send pull " << object_id << " request to node " << node_id - << " failed due to" << status.message(); - } - }); } void ObjectManager::HandlePushTaskTimeout(const ObjectID &object_id, @@ -528,14 +427,13 @@ void ObjectManager::SendObjectChunk(const UniqueID &push_id, const ObjectID &obj } void ObjectManager::CancelPull(const ObjectID &object_id) { - auto it = pull_requests_.find(object_id); - if (it == pull_requests_.end()) { + if (!pull_manager_->CancelPull(object_id)) { + // We weren't tracking a pull request for this object, so there is nothing to cancel. return; } RAY_CHECK_OK(object_directory_->UnsubscribeObjectLocations( object_directory_pull_callback_id_, object_id)); - pull_requests_.erase(it); } ray::Status ObjectManager::Wait( @@ -898,7 +796,7 @@ std::string ObjectManager::DebugString() const { result << "\n- num local objects: " << local_objects_.size(); result << "\n- num active wait requests: " << active_wait_requests_.size(); result << "\n- num unfulfilled push requests: " << unfulfilled_push_requests_.size(); - result << "\n- num pull requests: " << pull_requests_.size(); + result << "\n- num pull requests: " << pull_manager_->NumActiveRequests(); result << "\n- num buffered profile events: " << profile_events_.size(); result << "\n- num chunks received total: " << num_chunks_received_total_; result << "\n- num chunks received failed: " << num_chunks_received_failed_; @@ -913,7 +811,9 @@ void ObjectManager::RecordMetrics() const { stats::ObjectStoreAvailableMemory().Record(config_.object_store_memory - used_memory_); stats::ObjectStoreUsedMemory().Record(used_memory_); stats::ObjectStoreLocalObjects().Record(local_objects_.size()); - stats::ObjectManagerPullRequests().Record(pull_requests_.size()); + stats::ObjectManagerPullRequests().Record(pull_manager_->NumActiveRequests()); } +void ObjectManager::Tick() { pull_manager_->Tick(); } + } // namespace ray diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index d7d630c01..75dbb5bd4 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -32,12 +32,14 @@ #include "ray/common/id.h" #include "ray/common/ray_config.h" #include "ray/common/status.h" +#include "ray/object_manager/common.h" #include "ray/object_manager/format/object_manager_generated.h" #include "ray/object_manager/notification/object_store_notification_manager_ipc.h" #include "ray/object_manager/object_buffer_pool.h" #include "ray/object_manager/object_directory.h" #include "ray/object_manager/ownership_based_object_directory.h" #include "ray/object_manager/plasma/store_runner.h" +#include "ray/object_manager/pull_manager.h" #include "ray/object_manager/push_manager.h" #include "ray/rpc/object_manager/object_manager_client.h" #include "ray/rpc/object_manager/object_manager_server.h" @@ -49,6 +51,8 @@ struct ObjectManagerConfig { /// from other object managers. If this is 0, the object manager will choose /// its own port. int object_manager_port; + /// The object manager's global timer frequency. + unsigned int timer_freq_ms; /// The time in milliseconds to wait before retrying a pull /// that fails due to node id lookup. unsigned int pull_timeout_ms; @@ -78,7 +82,6 @@ struct LocalObjectInfo { /// Information from the object store about the object. object_manager::protocol::ObjectInfoT object_info; }; - class ObjectStoreRunner { public: ObjectStoreRunner(const ObjectManagerConfig &config, @@ -171,9 +174,8 @@ class ObjectManager : public ObjectManagerInterface, /// Send pull request /// /// \param object_id Object id - /// \param node_id Remote server node id - void SendPullRequest(const ObjectID &object_id, const NodeID &node_id, - std::shared_ptr rpc_client); + /// \param client_id Remote server client id + void SendPullRequest(const ObjectID &object_id, const NodeID &client_id); /// Get the rpc client according to the node ID /// @@ -235,17 +237,6 @@ class ObjectManager : public ObjectManagerInterface, /// \return Status of whether the pull request successfully initiated. ray::Status Pull(const ObjectID &object_id, const rpc::Address &owner_address) override; - /// Try to Pull an object from one of its expected node locations. If there - /// are more node locations to try after this attempt, then this method - /// will try each of the other nodes in succession, with a timeout between - /// each attempt. If the object is received or if the Pull is Canceled before - /// the timeout, then no more Pull requests for this object will be sent - /// to other node managers until TryPull is called again. - /// - /// \param object_id The object's object id. - /// \return Void. - void TryPull(const ObjectID &object_id); - /// Cancels all requests (Push/Pull) associated with the given ObjectID. This /// method is idempotent. /// @@ -293,16 +284,11 @@ class ObjectManager : public ObjectManagerInterface, /// Record metrics. void RecordMetrics() const; + void Tick(); + private: friend class TestObjectManager; - struct PullRequest { - PullRequest() : retry_timer(nullptr), timer_set(false), node_locations() {} - std::unique_ptr retry_timer; - bool timer_set; - std::vector node_locations; - }; - struct WaitState { WaitState(boost::asio::io_service &service, int64_t timeout_ms, const WaitCallback &callback) @@ -406,6 +392,10 @@ class ObjectManager : public ObjectManagerInterface, /// Handle Push task timeout. void HandlePushTaskTimeout(const ObjectID &object_id, const NodeID &node_id); + /// Weak reference to main service. We ensure this object is destroyed before + /// main_service_ is stopped. + boost::asio::io_service *main_service_; + NodeID self_node_id_; const ObjectManagerConfig config_; std::shared_ptr object_directory_; @@ -416,10 +406,6 @@ class ObjectManager : public ObjectManagerInterface, std::shared_ptr store_notification_; ObjectBufferPool buffer_pool_; - /// Weak reference to main service. We ensure this object is destroyed before - /// main_service_ is stopped. - boost::asio::io_service *main_service_; - /// Multi-thread asio service, deal with all outgoing and incoming RPC request. boost::asio::io_service rpc_service_; @@ -448,10 +434,6 @@ class ObjectManager : public ObjectManagerInterface, ObjectID, std::unordered_map>> unfulfilled_push_requests_; - /// The objects that this object manager is currently trying to fetch from - /// remote object managers. - std::unordered_map pull_requests_; - /// Profiling events that are to be batched together and added to the profile /// table in the GCS. std::vector profile_events_; @@ -460,9 +442,6 @@ class ObjectManager : public ObjectManagerInterface, /// and rpc thread. std::mutex profile_mutex_; - /// Internally maintained random number generator. - std::mt19937_64 gen_; - /// The gPRC server. rpc::GrpcServer object_manager_server_; @@ -478,9 +457,16 @@ class ObjectManager : public ObjectManagerInterface, const RestoreSpilledObjectCallback restore_spilled_object_; + /// Pull manager retry timer . + /* std::unique_ptr pull_retry_timer_; */ + boost::asio::deadline_timer pull_retry_timer_; + /// Object push manager. std::unique_ptr push_manager_; + /// Object pull manager. + std::unique_ptr pull_manager_; + /// Running sum of the amount of memory used in the object store. int64_t used_memory_ = 0; diff --git a/src/ray/object_manager/plasma/store.cc b/src/ray/object_manager/plasma/store.cc index 21f50161e..1ff2dd29d 100644 --- a/src/ray/object_manager/plasma/store.cc +++ b/src/ray/object_manager/plasma/store.cc @@ -133,6 +133,8 @@ PlasmaStore::PlasmaStore(boost::asio::io_service &main_service, std::string dire external_store_(external_store), spill_objects_callback_(spill_objects_callback), delay_on_oom_ms_(delay_on_oom_ms), + usage_log_interval_ns_(RayConfig::instance().object_store_usage_log_interval_s() * + 1e9), create_request_queue_( RayConfig::instance().object_store_full_max_retries(), /*evict_if_full=*/RayConfig::instance().object_pinning_enabled(), @@ -258,6 +260,13 @@ uint8_t *PlasmaStore::AllocateMemory(size_t size, bool evict_if_full, MEMFD_TYPE RAY_CHECK(*fd != INVALID_FD); *error = PlasmaError::OK; } + + auto now = absl::GetCurrentTimeNanos(); + if (now - last_usage_log_ns_ > usage_log_interval_ns_) { + RAY_LOG(INFO) << "Object store current usage " << (PlasmaAllocator::Allocated() / 1e9) + << " / " << (PlasmaAllocator::GetFootprintLimit() / 1e9) << " GB."; + last_usage_log_ns_ = now; + } return pointer; } diff --git a/src/ray/object_manager/plasma/store.h b/src/ray/object_manager/plasma/store.h index 4ea4ab51f..5ef2cd654 100644 --- a/src/ray/object_manager/plasma/store.h +++ b/src/ray/object_manager/plasma/store.h @@ -296,6 +296,12 @@ class PlasmaStore { /// transient OOM error. const uint32_t delay_on_transient_oom_ms_ = 10; + /// The amount of time to wait between logging space usage debug messages. + const uint64_t usage_log_interval_ns_; + + /// The last time space usage was logged. + uint64_t last_usage_log_ns_ = 0; + /// A timer that is set when the first request in the queue is not /// serviceable because there is not enough memory. The request will be /// retried when this timer expires. diff --git a/src/ray/object_manager/pull_manager.cc b/src/ray/object_manager/pull_manager.cc new file mode 100644 index 000000000..082426cc1 --- /dev/null +++ b/src/ray/object_manager/pull_manager.cc @@ -0,0 +1,139 @@ +#include "ray/object_manager/pull_manager.h" + +namespace ray { + +PullManager::PullManager( + NodeID &self_node_id, const std::function object_is_local, + const std::function send_pull_request, + const RestoreSpilledObjectCallback restore_spilled_object, + const std::function get_time, int pull_timeout_ms) + : self_node_id_(self_node_id), + object_is_local_(object_is_local), + send_pull_request_(send_pull_request), + restore_spilled_object_(restore_spilled_object), + get_time_(get_time), + pull_timeout_ms_(pull_timeout_ms), + gen_(std::chrono::high_resolution_clock::now().time_since_epoch().count()) {} + +bool PullManager::Pull(const ObjectID &object_id, const rpc::Address &owner_address) { + RAY_LOG(DEBUG) << "Pull " + << " of object " << object_id; + // Check if object is already local. + if (object_is_local_(object_id)) { + RAY_LOG(DEBUG) << object_id << " attempted to pull an object that's already local."; + return false; + } + if (pull_requests_.find(object_id) != pull_requests_.end()) { + RAY_LOG(DEBUG) << object_id << " has inflight pull_requests, skipping."; + return false; + } + + pull_requests_.emplace(object_id, PullRequest(get_time_() + pull_timeout_ms_ / 1000)); + return true; +} + +void PullManager::OnLocationChange(const ObjectID &object_id, + const std::unordered_set &client_ids, + const std::string &spilled_url) { + // Exit if the Pull request has already been fulfilled or canceled. + auto it = pull_requests_.find(object_id); + if (it == pull_requests_.end()) { + return; + } + // Reset the list of clients that are now expected to have the object. + // NOTE(swang): Since we are overwriting the previous list of clients, + // we may end up sending a duplicate request to the same client as + // before. + it->second.client_locations = std::vector(client_ids.begin(), client_ids.end()); + if (!spilled_url.empty()) { + // Try to restore the spilled object. + restore_spilled_object_(object_id, spilled_url, + [this, object_id](const ray::Status &status) { + // Fall back to fetching from another object manager. + if (!status.ok()) { + TryPull(object_id); + } + }); + } else { + // New object locations were found, so begin trying to pull from a + // client. This will be called every time a new client location + // appears. + TryPull(object_id); + } +} + +void PullManager::TryPull(const ObjectID &object_id) { + auto it = pull_requests_.find(object_id); + if (it == pull_requests_.end()) { + return; + } + + auto &node_vector = it->second.client_locations; + + // The timer should never fire if there are no expected client locations. + if (node_vector.empty()) { + return; + } + + RAY_CHECK(!object_is_local_(object_id)); + // Make sure that there is at least one client which is not the local client. + // TODO(rkn): It may actually be possible for this check to fail. + if (node_vector.size() == 1 && node_vector[0] == self_node_id_) { + RAY_LOG(WARNING) << "The object manager with ID " << self_node_id_ + << " is trying to pull object " << object_id + << " but the object table suggests that this object manager " + << "already has the object. The object may have been evicted. It is " + << "most likely due to memory pressure, object pull has been " + << "requested before object location is updated."; + return; + } + + // Choose a random client to pull the object from. + // Generate a random index. + std::uniform_int_distribution distribution(0, node_vector.size() - 1); + int node_index = distribution(gen_); + NodeID node_id = node_vector[node_index]; + // If the object manager somehow ended up choosing itself, choose a different + // object manager. + if (node_id == self_node_id_) { + std::swap(node_vector[node_index], node_vector[node_vector.size() - 1]); + node_vector.pop_back(); + RAY_LOG(WARNING) + << "The object manager with ID " << self_node_id_ << " is trying to pull object " + << object_id << " but the object table suggests that this object manager " + << "already has the object. It is most likely due to memory pressure, object " + << "pull has been requested before object location is updated."; + node_id = node_vector[node_index % node_vector.size()]; + RAY_CHECK(node_id != self_node_id_); + } + + RAY_LOG(DEBUG) << "Sending pull request from " << self_node_id_ << " to " << node_id + << " of object " << object_id; + send_pull_request_(object_id, node_id); +} + +bool PullManager::CancelPull(const ObjectID &object_id) { + auto it = pull_requests_.find(object_id); + if (it == pull_requests_.end()) { + return false; + } + + pull_requests_.erase(it); + return true; +} + +void PullManager::Tick() { + for (auto &pair : pull_requests_) { + const auto &object_id = pair.first; + auto &request = pair.second; + const auto time = get_time_(); + if (time >= request.next_pull_time) { + TryPull(object_id); + request.next_pull_time = time + pull_timeout_ms_ / 1000; + } + } +} + +int PullManager::NumActiveRequests() const { return pull_requests_.size(); } + +} // namespace ray diff --git a/src/ray/object_manager/pull_manager.h b/src/ray/object_manager/pull_manager.h new file mode 100644 index 000000000..023f72d0e --- /dev/null +++ b/src/ray/object_manager/pull_manager.h @@ -0,0 +1,115 @@ + +#pragma once + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/time/clock.h" +#include "ray/common/id.h" +#include "ray/common/ray_config.h" +#include "ray/common/status.h" +#include "ray/object_manager/common.h" +#include "ray/object_manager/format/object_manager_generated.h" +#include "ray/object_manager/notification/object_store_notification_manager_ipc.h" +#include "ray/object_manager/object_buffer_pool.h" +#include "ray/object_manager/object_directory.h" +#include "ray/object_manager/ownership_based_object_directory.h" +#include "ray/object_manager/plasma/store_runner.h" +#include "ray/rpc/object_manager/object_manager_client.h" +#include "ray/rpc/object_manager/object_manager_server.h" + +namespace ray { + +class PullManager { + public: + /// PullManager is responsible for managing the policy around when to send pull requests + /// and to whom. Notably, it is _not_ responsible for controlling the object directory + /// or any pubsub communications. + /// + /// \param self_node_id the current node + /// \param object_is_local A callback which should return true if a given object is + /// already on the local node. \param send_pull_request A callback which should send a + /// pull request to the specified node. + /// \param restore_spilled_object A callback which should + /// retrieve an spilled object from the external store. + PullManager( + NodeID &self_node_id, const std::function object_is_local, + const std::function send_pull_request, + const RestoreSpilledObjectCallback restore_spilled_object, + const std::function get_time, int pull_timeout_ms); + + /// Begin a new pull request if necessary. + /// + /// \param object_id The object id to pull. + /// \param owner_address The owner of the object. + /// + /// \return True if a new pull request was necessary. If true, the caller should + /// subscribe to new locations of the object, and call OnLocationChange when necessary. + bool Pull(const ObjectID &object_id, const rpc::Address &owner_address); + + /// Called when the available locations for a given object change. + /// + /// \param object_id The ID of the object which is now available in a new location. + /// \param client_ids The new set of nodes that the object is available on. Not + /// necessarily a super or subset of the previously available nodes. \param spilled_url + /// The location of the object if it was spilled. If non-empty, the object may no longer + /// be on any node. + void OnLocationChange(const ObjectID &object_id, + const std::unordered_set &client_ids, + const std::string &spilled_url); + + /// Cancel an existing pull request if necessary. + /// + /// \param object_id The object id that no longer needs to be pulled. + /// + /// \return True if a pull was cancelled. If there was no pending pull request for the + /// object this method may return false. + bool CancelPull(const ObjectID &object_id); + + /// Called when the retry timer fires. If this fires, the pull manager may try to pull + /// existing objects from other nodes if necessary. + void Tick(); + + /// The number of ongoing object pulls. + int NumActiveRequests() const; + + private: + /// A helper structure for tracking information about each ongoing object pull. + struct PullRequest { + PullRequest(double first_retry_time) + : client_locations(), next_pull_time(first_retry_time) {} + std::vector client_locations; + double next_pull_time; + }; + + /// See the constructor's arguments. + NodeID self_node_id_; + const std::function object_is_local_; + const std::function send_pull_request_; + const RestoreSpilledObjectCallback restore_spilled_object_; + const std::function get_time_; + int pull_timeout_ms_; + + /// The objects that this object manager is currently trying to fetch from + /// remote object managers. + std::unordered_map pull_requests_; + + /// Internally maintained random number generator. + std::mt19937_64 gen_; + + /// Try to Pull an object from one of its expected client locations. If there + /// are more client locations to try after this attempt, then this method + /// will try each of the other clients in succession, with a timeout between + /// each attempt. If the object is received or if the Pull is Canceled before + /// the timeout, then no more Pull requests for this object will be sent + /// to other node managers until TryPull is called again. + /// + /// \param object_id The object's object id. + /// \return Void. + void TryPull(const ObjectID &object_id); +}; +} // namespace ray diff --git a/src/ray/object_manager/test/pull_manager_test.cc b/src/ray/object_manager/test/pull_manager_test.cc new file mode 100644 index 000000000..90f34048c --- /dev/null +++ b/src/ray/object_manager/test/pull_manager_test.cc @@ -0,0 +1,173 @@ + +#include "ray/object_manager/pull_manager.h" + +#include "gtest/gtest.h" +#include "ray/common/test_util.h" + +namespace ray { + +class PullManagerTest : public ::testing::Test { + public: + PullManagerTest() + : self_node_id_(NodeID::FromRandom()), + object_is_local_(false), + num_send_pull_request_calls_(0), + num_restore_spilled_object_calls_(0), + fake_time_(0), + pull_manager_(self_node_id_, + [this](const ObjectID &object_id) { return object_is_local_; }, + [this](const ObjectID &object_id, const NodeID &node_id) { + num_send_pull_request_calls_++; + }, + [this](const ObjectID &, const std::string &, + std::function) { + num_restore_spilled_object_calls_++; + }, + [this]() { return fake_time_; }, 10000) {} + + NodeID self_node_id_; + bool object_is_local_; + int num_send_pull_request_calls_; + int num_restore_spilled_object_calls_; + double fake_time_; + PullManager pull_manager_; +}; + +TEST_F(PullManagerTest, TestStaleSubscription) { + ObjectID obj1 = ObjectID::FromRandom(); + rpc::Address addr1; + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + pull_manager_.Pull(obj1, addr1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 1); + + std::unordered_set client_ids; + pull_manager_.OnLocationChange(obj1, client_ids, ""); + + // There are no client ids to pull from. + ASSERT_EQ(num_send_pull_request_calls_, 0); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + + pull_manager_.CancelPull(obj1); + + ASSERT_EQ(num_send_pull_request_calls_, 0); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + + client_ids.insert(NodeID::FromRandom()); + pull_manager_.OnLocationChange(obj1, client_ids, ""); + + // Now we're getting a notification about an object that was already cancelled. + ASSERT_EQ(num_send_pull_request_calls_, 0); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); +} + +TEST_F(PullManagerTest, TestRestoreSpilledObject) { + ObjectID obj1 = ObjectID::FromRandom(); + rpc::Address addr1; + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + pull_manager_.Pull(obj1, addr1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 1); + + std::unordered_set client_ids; + pull_manager_.OnLocationChange(obj1, client_ids, "remote_url/foo/bar"); + + // client_ids is empty here, so there's nowhere to pull from. + ASSERT_EQ(num_send_pull_request_calls_, 0); + ASSERT_EQ(num_restore_spilled_object_calls_, 1); + + client_ids.insert(NodeID::FromRandom()); + pull_manager_.OnLocationChange(obj1, client_ids, "remote_url/foo/bar"); + + // The behavior is supposed to be to always restore the spilled object if possible (even + // if it exists elsewhere in the cluster). + ASSERT_EQ(num_send_pull_request_calls_, 0); + ASSERT_EQ(num_restore_spilled_object_calls_, 2); + + pull_manager_.CancelPull(obj1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); +} + +TEST_F(PullManagerTest, TestManyUpdates) { + ObjectID obj1 = ObjectID::FromRandom(); + rpc::Address addr1; + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + pull_manager_.Pull(obj1, addr1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 1); + + std::unordered_set client_ids; + client_ids.insert(NodeID::FromRandom()); + + for (int i = 0; i < 100; i++) { + pull_manager_.OnLocationChange(obj1, client_ids, ""); + } + + ASSERT_EQ(num_send_pull_request_calls_, 100); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + + pull_manager_.CancelPull(obj1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); +} + +TEST_F(PullManagerTest, TestRetryTimer) { + ObjectID obj1 = ObjectID::FromRandom(); + rpc::Address addr1; + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + pull_manager_.Pull(obj1, addr1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 1); + + std::unordered_set client_ids; + client_ids.insert(NodeID::FromRandom()); + + // We need to call OnLocationChange at least once, to population the list of nodes with + // the object. + pull_manager_.OnLocationChange(obj1, client_ids, ""); + ASSERT_EQ(num_send_pull_request_calls_, 1); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + + for (; fake_time_ <= 127 * 10; fake_time_ += 0.1) { + pull_manager_.Tick(); + } + + // Rapid set of location changes. + for (int i = 0; i < 127; i++) { + fake_time_ += 0.1; + pull_manager_.OnLocationChange(obj1, client_ids, ""); + } + + // We should make a pull request every tick (even if it's a duplicate to a node we're + // already pulling from). + // OnLocationChange also doesn't count towards the retry timer. + // To the casual observer, this may seem off-by-one, but this is due to floating point + // error (0.1 + 0.1 ... 10k times > 10 == True) + ASSERT_EQ(num_send_pull_request_calls_, 127 * 2); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + + pull_manager_.CancelPull(obj1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); +} + +TEST_F(PullManagerTest, TestBasic) { + ObjectID obj1 = ObjectID::FromRandom(); + rpc::Address addr1; + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); + pull_manager_.Pull(obj1, addr1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 1); + + std::unordered_set client_ids; + client_ids.insert(NodeID::FromRandom()); + pull_manager_.OnLocationChange(obj1, client_ids, ""); + + ASSERT_EQ(num_send_pull_request_calls_, 1); + ASSERT_EQ(num_restore_spilled_object_calls_, 0); + + pull_manager_.CancelPull(obj1); + ASSERT_EQ(pull_manager_.NumActiveRequests(), 0); +} + +} // namespace ray + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index b894f92aa..d9337708d 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -200,6 +200,9 @@ message TaskSpec { // the receiver will not execute the task. This field is used by async actors // to guarantee task submission order after restart. bool skip_execution = 22; + // Breakpoint if this task should drop into the debugger when it starts executing + // and "" if the task should not drop into the debugger. + bytes debugger_breakpoint = 23; } message Bundle { @@ -255,24 +258,29 @@ message TaskArg { message ActorCreationTaskSpec { // ID of the actor that will be created by this task. bytes actor_id = 2; - // The max number of times this actor should be recontructed. + // The max number of times this actor should be restarted. // If this number is 0 the actor won't be restarted. // If this number is -1 the actor will be restarted indefinitely. int64 max_actor_restarts = 3; + // The max number of times tasks submitted on this actor should be retried + // if the actor fails and is restarted. + // If this number is 0 the tasks won't be resubmitted. + // If this number is -1 the tasks will be resubmitted indefinitely. + int64 max_task_retries = 4; // The dynamic options used in the worker command when starting a worker process for // an actor creation task. If the list isn't empty, the options will be used to replace // the placeholder string `RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER` in the worker command. - repeated string dynamic_worker_options = 4; + repeated string dynamic_worker_options = 5; // The max number of concurrent calls for direct call actors. - int32 max_concurrency = 5; + int32 max_concurrency = 6; // Whether the actor is persistent. - bool is_detached = 6; + bool is_detached = 7; // Globally-unique name of the actor. Should only be populated when is_detached is true. - string name = 7; + string name = 8; // Whether the actor use async actor calls. - bool is_asyncio = 8; + bool is_asyncio = 9; // Field used for storing application-level extensions to the actor definition. - string extension_data = 9; + string extension_data = 10; } // Task spec of an actor task. diff --git a/src/ray/protobuf/gcs.proto b/src/ray/protobuf/gcs.proto index 580623185..fe2511bd0 100644 --- a/src/ray/protobuf/gcs.proto +++ b/src/ray/protobuf/gcs.proto @@ -32,7 +32,7 @@ enum TablePrefix { FUNCTION = 7; TASK_RECONSTRUCTION = 8; HEARTBEAT = 9; - HEARTBEAT_BATCH = 10; + RESOURCE_USAGE_BATCH = 10; JOB = 11; PROFILE = 12; TASK_LEASE = 13; @@ -56,7 +56,7 @@ enum TablePubsub { OBJECT_PUBSUB = 5; ACTOR_PUBSUB = 6; HEARTBEAT_PUBSUB = 7; - HEARTBEAT_BATCH_PUBSUB = 8; + RESOURCE_USAGE_BATCH_PUBSUB = 8; TASK_LEASE_PUBSUB = 9; JOB_PUBSUB = 10; NODE_RESOURCE_PUBSUB = 11; @@ -302,22 +302,6 @@ message PlacementGroupLoad { message HeartbeatTableData { // Node id. bytes node_id = 1; - // Resource capacity currently available on this node manager. - map resources_available = 2; - // Indicates whether avaialbe resources is changed. Only used when - // light heartbeat enabled. - bool resources_available_changed = 3; - // Total resource capacity configured for this node manager. - map resources_total = 4; - // Aggregate outstanding resource load on this node manager. - map resource_load = 5; - // Indicates whether resource load is changed. Only used when - // light heartbeat enabled. - bool resource_load_changed = 6; - // The resource load on this node, sorted by resource shape. - ResourceLoad resource_load_by_shape = 7; - // Whether this node manager is requesting global GC. - bool should_global_gc = 8; } message ResourcesData { @@ -325,8 +309,8 @@ message ResourcesData { bytes node_id = 1; // Resource capacity currently available on this node manager. map resources_available = 2; - // Indicates whether avaialbe resources is changed. Only used when - // light heartbeat enabled. + // Indicates whether available resources is changed. Only used when light + // heartbeat enabled. bool resources_available_changed = 3; // Total resource capacity configured for this node manager. map resources_total = 4; @@ -341,8 +325,8 @@ message ResourcesData { bool should_global_gc = 8; } -message HeartbeatBatchTableData { - repeated HeartbeatTableData batch = 1; +message ResourceUsageBatchData { + repeated ResourcesData batch = 1; // The total resource demand on all nodes included in the batch, sorted by // resource shape. ResourceLoad resource_load_by_shape = 2; diff --git a/src/ray/protobuf/gcs_service.proto b/src/ray/protobuf/gcs_service.proto index a68264359..eb730a7cf 100644 --- a/src/ray/protobuf/gcs_service.proto +++ b/src/ray/protobuf/gcs_service.proto @@ -164,14 +164,6 @@ message ReportHeartbeatReply { GcsStatus status = 1; } -message GetAllHeartbeatRequest { -} - -message GetAllHeartbeatReply { - GcsStatus status = 1; - HeartbeatBatchTableData heartbeat_data = 2; -} - message ReportResourceUsageRequest { ResourcesData resources = 1; } @@ -185,7 +177,41 @@ message GetAllResourceUsageRequest { message GetAllResourceUsageReply { GcsStatus status = 1; - repeated ResourcesData resources_list = 2; + ResourceUsageBatchData resource_usage_data = 2; +} + +message SetInternalConfigRequest { + StoredConfig config = 1; +} + +message SetInternalConfigReply { + GcsStatus status = 1; +} + +message GetInternalConfigRequest { +} + +message GetInternalConfigReply { + GcsStatus status = 1; + StoredConfig config = 2; +} + +// Service for node info access. +service NodeInfoGcsService { + // Register a node to GCS Service. + rpc RegisterNode(RegisterNodeRequest) returns (RegisterNodeReply); + // Unregister a node from GCS Service. + rpc UnregisterNode(UnregisterNodeRequest) returns (UnregisterNodeReply); + // Get information of all nodes from GCS Service. + rpc GetAllNodeInfo(GetAllNodeInfoRequest) returns (GetAllNodeInfoReply); + // Report resource usage of a node to GCS Service. + rpc ReportResourceUsage(ReportResourceUsageRequest) returns (ReportResourceUsageReply); + // Get resource usage of all nodes from GCS Service. + rpc GetAllResourceUsage(GetAllResourceUsageRequest) returns (GetAllResourceUsageReply); + // Set cluster internal config. + rpc SetInternalConfig(SetInternalConfigRequest) returns (SetInternalConfigReply); + // Get cluster internal config. + rpc GetInternalConfig(GetInternalConfigRequest) returns (GetInternalConfigReply); } message GetResourcesRequest { @@ -215,22 +241,6 @@ message DeleteResourcesReply { GcsStatus status = 1; } -message SetInternalConfigRequest { - StoredConfig config = 1; -} - -message SetInternalConfigReply { - GcsStatus status = 1; -} - -message GetInternalConfigRequest { -} - -message GetInternalConfigReply { - GcsStatus status = 1; - StoredConfig config = 2; -} - message GetAllAvailableResourcesRequest { } @@ -239,37 +249,25 @@ message GetAllAvailableResourcesReply { repeated AvailableResources resources_list = 2; } -// Service for node info access. -service NodeInfoGcsService { - // Register a node to GCS Service. - rpc RegisterNode(RegisterNodeRequest) returns (RegisterNodeReply); - // Unregister a node from GCS Service. - rpc UnregisterNode(UnregisterNodeRequest) returns (UnregisterNodeReply); - // Get information of all nodes from GCS Service. - rpc GetAllNodeInfo(GetAllNodeInfoRequest) returns (GetAllNodeInfoReply); - // Report heartbeat of a node to GCS Service. - rpc ReportHeartbeat(ReportHeartbeatRequest) returns (ReportHeartbeatReply); - // Get newest heartbeat of all nodes from GCS Service. - rpc GetAllHeartbeat(GetAllHeartbeatRequest) returns (GetAllHeartbeatReply); - // Report resource usage of a node to GCS Service. - rpc ReportResourceUsage(ReportResourceUsageRequest) returns (ReportResourceUsageReply); - // Get resource usage of all nodes from GCS Service. - rpc GetAllResourceUsage(GetAllResourceUsageRequest) returns (GetAllResourceUsageReply); +// Service for node resource info access. +service NodeResourceInfoGcsService { // Get node's resources from GCS Service. rpc GetResources(GetResourcesRequest) returns (GetResourcesReply); // Update resources of a node in GCS Service. rpc UpdateResources(UpdateResourcesRequest) returns (UpdateResourcesReply); // Delete resources of a node in GCS Service. rpc DeleteResources(DeleteResourcesRequest) returns (DeleteResourcesReply); - // Set cluster internal config. - rpc SetInternalConfig(SetInternalConfigRequest) returns (SetInternalConfigReply); - // Get cluster internal config. - rpc GetInternalConfig(GetInternalConfigRequest) returns (GetInternalConfigReply); // Get available resources of all nodes. rpc GetAllAvailableResources(GetAllAvailableResourcesRequest) returns (GetAllAvailableResourcesReply); } +// Service for heartbeat info access. +service HeartbeatInfoGcsService { + // Report heartbeat of a node to GCS Service. + rpc ReportHeartbeat(ReportHeartbeatRequest) returns (ReportHeartbeatReply); +} + message GetObjectLocationsRequest { // The ID of object to lookup in GCS Service. bytes object_id = 1; @@ -527,20 +525,28 @@ message GetAllPlacementGroupReply { repeated PlacementGroupTableData placement_group_table_data = 2; } +message WaitPlacementGroupUntilReadyRequest { + bytes placement_group_id = 1; +} + +message WaitPlacementGroupUntilReadyReply { + GcsStatus status = 1; +} + // Service for placement group info access. service PlacementGroupInfoGcsService { // Create placement group via gcs service. rpc CreatePlacementGroup(CreatePlacementGroupRequest) returns (CreatePlacementGroupReply); - // Remove placement group via gcs service. rpc RemovePlacementGroup(RemovePlacementGroupRequest) returns (RemovePlacementGroupReply); - // Get placement group information via gcs service. rpc GetPlacementGroup(GetPlacementGroupRequest) returns (GetPlacementGroupReply); - // Get information of all placement group from GCS Service. rpc GetAllPlacementGroup(GetAllPlacementGroupRequest) returns (GetAllPlacementGroupReply); -} \ No newline at end of file + // Wait for placement group until ready. + rpc WaitPlacementGroupUntilReady(WaitPlacementGroupUntilReadyRequest) + returns (WaitPlacementGroupUntilReadyReply); +} diff --git a/src/ray/protobuf/ray_client.proto b/src/ray/protobuf/ray_client.proto index fd8fe5345..d4c392321 100644 --- a/src/ray/protobuf/ray_client.proto +++ b/src/ray/protobuf/ray_client.proto @@ -42,8 +42,13 @@ message ClientTask { repeated Arg args = 4; } +message RemoteRef { + bytes id = 1; + bytes handle = 2; +} + message ClientTaskTicket { - bytes return_id = 1; + RemoteRef return_ref = 1; } message PutRequest { @@ -51,27 +56,75 @@ message PutRequest { } message PutResponse { - bytes id = 1; + RemoteRef ref = 1; } message GetRequest { - bytes id = 1; + bytes handle = 1; + float timeout = 2; } message GetResponse { bool valid = 1; bytes data = 2; } + message WaitRequest { - repeated bytes object_refs = 1; + repeated bytes object_handles = 1; int64 num_returns = 2; double timeout = 3; } message WaitResponse { bool valid = 1; - repeated bytes ready_object_ids = 2; - repeated bytes remaining_object_ids = 3; + repeated RemoteRef ready_object_ids = 2; + repeated RemoteRef remaining_object_ids = 3; +} + +message ClusterInfoType { + // Namespace the enum, as it collides in the overall package. + enum TypeEnum { + IS_INITIALIZED = 0; + NODES = 1; + CLUSTER_RESOURCES = 2; + AVAILABLE_RESOURCES = 3; + } +} + +message ClusterInfoRequest { + ClusterInfoType.TypeEnum type = 1; +} + +message ClusterInfoResponse { + message ResourceTable { + map table = 1; + } + ClusterInfoType.TypeEnum type = 1; + oneof response_type { + string json = 2; + ResourceTable resource_table = 3; + } +} + +message TerminateRequest { + message ActorTerminate { + bytes handle = 1; + bool no_restart = 2; + } + message TaskObjectTerminate { + bytes handle = 1; + bool force = 2; + bool recursive = 3; + } + + oneof terminate_type { + ActorTerminate actor = 1; + TaskObjectTerminate task_object = 2; + } +} + +message TerminateResponse { + bool ok = 1; } service RayletDriver { @@ -83,4 +136,8 @@ service RayletDriver { } rpc Schedule(ClientTask) returns (ClientTaskTicket) { } + rpc Terminate(TerminateRequest) returns (TerminateResponse) { + } + rpc ClusterInfo(ClusterInfoRequest) returns (ClusterInfoResponse) { + } } diff --git a/src/ray/raylet/agent_manager.cc b/src/ray/raylet/agent_manager.cc index 23b8769c8..7445c7034 100644 --- a/src/ray/raylet/agent_manager.cc +++ b/src/ray/raylet/agent_manager.cc @@ -60,6 +60,7 @@ void AgentManager::StartAgent() { // Set node id to agent. ProcessEnvironment env; env.insert({"RAY_NODE_ID", options_.node_id.Hex()}); + env.insert({"RAY_RAYLET_PID", std::to_string(getpid())}); Process child(argv.data(), nullptr, ec, false, env); if (!child.IsValid() || ec) { // The worker failed to start. This is a fatal error. diff --git a/src/ray/raylet/local_object_manager.cc b/src/ray/raylet/local_object_manager.cc index bd0ca72b1..b42641a1e 100644 --- a/src/ray/raylet/local_object_manager.cc +++ b/src/ray/raylet/local_object_manager.cc @@ -52,7 +52,7 @@ void LocalObjectManager::WaitForObjectFree(const rpc::Address &owner_address, wait_request, [this, object_id](Status status, const rpc::WaitForObjectEvictionReply &reply) { if (!status.ok()) { - RAY_LOG(WARNING) << "Worker failed. Unpinning object " << object_id; + RAY_LOG(DEBUG) << "Worker failed. Unpinning object " << object_id; } ReleaseFreedObject(object_id); }); diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index a8e5ad5ca..4b92d7163 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -38,7 +38,6 @@ DEFINE_int32(max_worker_port, 0, "The highest port that workers' gRPC servers will bind on."); DEFINE_string(worker_port_list, "", "An explicit list of ports that workers' gRPC servers will bind on."); -DEFINE_int32(num_initial_workers, 0, "Number of initial workers."); DEFINE_int32(num_initial_python_workers_for_first_job, 0, "Number of initial Python workers for the first job."); DEFINE_int32(maximum_startup_concurrency, 1, "Maximum startup concurrency"); @@ -78,7 +77,6 @@ int main(int argc, char *argv[]) { const int min_worker_port = static_cast(FLAGS_min_worker_port); const int max_worker_port = static_cast(FLAGS_max_worker_port); const std::string worker_port_list = FLAGS_worker_port_list; - const int num_initial_workers = static_cast(FLAGS_num_initial_workers); const int num_initial_python_workers_for_first_job = static_cast(FLAGS_num_initial_python_workers_for_first_job); const int maximum_startup_concurrency = @@ -183,7 +181,6 @@ int main(int argc, char *argv[]) { << node_manager_config.resource_config.ToString(); node_manager_config.node_manager_address = node_ip_address; node_manager_config.node_manager_port = node_manager_port; - node_manager_config.num_initial_workers = num_initial_workers; node_manager_config.num_workers_soft_limit = num_cpus; node_manager_config.num_initial_python_workers_for_first_job = num_initial_python_workers_for_first_job; @@ -217,6 +214,8 @@ int main(int argc, char *argv[]) { node_manager_config.heartbeat_period_ms = RayConfig::instance().raylet_heartbeat_timeout_milliseconds(); + node_manager_config.report_resources_period_ms = + RayConfig::instance().raylet_report_resources_period_milliseconds(); node_manager_config.debug_dump_period_ms = RayConfig::instance().debug_dump_period_milliseconds(); node_manager_config.record_metrics_period_ms = @@ -235,6 +234,9 @@ int main(int argc, char *argv[]) { ray::ObjectManagerConfig object_manager_config; object_manager_config.object_manager_port = object_manager_port; object_manager_config.store_socket_name = store_socket_name; + + object_manager_config.timer_freq_ms = + RayConfig::instance().object_manager_timer_freq_ms(); object_manager_config.pull_timeout_ms = RayConfig::instance().object_manager_pull_timeout_ms(); object_manager_config.push_timeout_ms = @@ -258,7 +260,7 @@ int main(int argc, char *argv[]) { // Initialize stats. const ray::stats::TagsType global_tags = { {ray::stats::ComponentKey, "raylet"}, - {ray::stats::VersionKey, "1.1.0.dev0"}, + {ray::stats::VersionKey, "1.2.0.dev0"}, {ray::stats::NodeAddressKey, node_ip_address}}; ray::stats::Init(global_tags, metrics_agent_port); diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index f20956cea..31ef907e0 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -124,22 +124,31 @@ NodeManager::NodeManager(boost::asio::io_service &io_service, const NodeID &self object_directory_(object_directory), heartbeat_timer_(io_service), heartbeat_period_(std::chrono::milliseconds(config.heartbeat_period_ms)), + report_resources_timer_(io_service), + report_resources_period_( + std::chrono::milliseconds(config.report_resources_period_ms)), debug_dump_period_(config.debug_dump_period_ms), fair_queueing_enabled_(config.fair_queueing_enabled), object_pinning_enabled_(config.object_pinning_enabled), temp_dir_(config.temp_dir), object_manager_profile_timer_(io_service), - light_heartbeat_enabled_(RayConfig::instance().light_heartbeat_enabled()), + light_report_resource_usage_enabled_( + RayConfig::instance().light_report_resource_usage_enabled()), initial_config_(config), local_available_resources_(config.resource_config), - worker_pool_( - io_service, config.num_initial_workers, config.num_workers_soft_limit, - config.num_initial_python_workers_for_first_job, - config.maximum_startup_concurrency, config.min_worker_port, - config.max_worker_port, config.worker_ports, gcs_client_, - config.worker_commands, config.raylet_config, - /*starting_worker_timeout_callback=*/ - [this]() { this->DispatchTasks(this->local_queues_.GetReadyTasksByClass()); }), + worker_pool_(io_service, config.num_workers_soft_limit, + config.num_initial_python_workers_for_first_job, + config.maximum_startup_concurrency, config.min_worker_port, + config.max_worker_port, config.worker_ports, gcs_client_, + config.worker_commands, config.raylet_config, + /*starting_worker_timeout_callback=*/ + [this]() { + if (RayConfig::instance().new_scheduler_enabled()) { + ScheduleAndDispatch(); + } else { + this->DispatchTasks(this->local_queues_.GetReadyTasksByClass()); + } + }), scheduling_policy_(local_queues_), reconstruction_policy_( io_service_, @@ -168,7 +177,11 @@ NodeManager::NodeManager(boost::asio::io_service &io_service, const NodeID &self }), new_scheduler_enabled_(RayConfig::instance().new_scheduler_enabled()), report_worker_backlog_(RayConfig::instance().report_worker_backlog()), + last_local_gc_ns_(absl::GetCurrentTimeNanos()), + local_gc_interval_ns_(RayConfig::instance().local_gc_interval_s() * 1e9), record_metrics_period_(config.record_metrics_period_ms) { + placement_group_resource_manager_ = std::make_shared( + local_available_resources_, cluster_resource_map_, self_node_id_); RAY_LOG(INFO) << "Initializing NodeManager with ID " << self_node_id_; RAY_CHECK(heartbeat_period_.count() > 0); // Initialize the resource map with own cluster resource configuration. @@ -260,7 +273,7 @@ ray::Status NodeManager::RegisterGcs() { id, VectorFromProtobuf(resource_notification.deleted_resources())); } }; - RAY_CHECK_OK(gcs_client_->Nodes().AsyncSubscribeToResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncSubscribeToResources( /*subscribe_callback=*/resources_changed, /*done_callback=*/nullptr)); }; @@ -268,13 +281,13 @@ ray::Status NodeManager::RegisterGcs() { RAY_RETURN_NOT_OK( gcs_client_->Nodes().AsyncSubscribeToNodeChange(on_node_change, on_done)); - // Subscribe to heartbeat batches from the monitor. - const auto &heartbeat_batch_added = - [this](const HeartbeatBatchTableData &heartbeat_batch) { - HeartbeatBatchAdded(heartbeat_batch); + // Subscribe to resource usage batches from the monitor. + const auto &resource_usage_batch_added = + [this](const ResourceUsageBatchData &resource_usage_batch) { + ResourceUsageBatchAdded(resource_usage_batch); }; - RAY_RETURN_NOT_OK(gcs_client_->Nodes().AsyncSubscribeBatchHeartbeat( - heartbeat_batch_added, /*done*/ nullptr)); + RAY_RETURN_NOT_OK(gcs_client_->Nodes().AsyncSubscribeBatchedResourceUsage( + resource_usage_batch_added, /*done*/ nullptr)); // Subscribe to all unexpected failure notifications from the local and // remote raylets. Note that this does not include workers that failed due to @@ -303,6 +316,7 @@ ray::Status NodeManager::RegisterGcs() { last_heartbeat_at_ms_ = current_time_ms(); last_debug_dump_at_ms_ = current_time_ms(); Heartbeat(); + ReportResourceUsage(); // Start the timer that gets object manager profiling information and sends it // to the GCS. GetObjectManagerProfileInfo(); @@ -345,15 +359,13 @@ void NodeManager::HandleJobStarted(const JobID &job_id, const JobTableData &job_ RAY_CHECK(!job_data.is_dead()); worker_pool_.HandleJobStarted(job_id, job_data.config()); - if (RayConfig::instance().enable_multi_tenancy()) { - // Tasks of this job may already arrived but failed to pop a worker because the job - // config is not local yet. So we trigger dispatching again here to try to - // reschedule these tasks. - if (new_scheduler_enabled_) { - ScheduleAndDispatch(); - } else { - DispatchTasks(local_queues_.GetReadyTasksByClass()); - } + // Tasks of this job may already arrived but failed to pop a worker because the job + // config is not local yet. So we trigger dispatching again here to try to + // reschedule these tasks. + if (new_scheduler_enabled_) { + ScheduleAndDispatch(); + } else { + DispatchTasks(local_queues_.GetReadyTasksByClass()); } } @@ -404,99 +416,13 @@ void NodeManager::Heartbeat() { stats::HeartbeatReportMs.Record(interval); auto heartbeat_data = std::make_shared(); - SchedulingResources &local_resources = cluster_resource_map_[self_node_id_]; heartbeat_data->set_node_id(self_node_id_.Binary()); - - if (new_scheduler_enabled_) { - new_resource_scheduler_->Heartbeat(light_heartbeat_enabled_, heartbeat_data); - cluster_task_manager_->Heartbeat(light_heartbeat_enabled_, heartbeat_data); - } else { - // TODO(atumanov): modify the heartbeat table protocol to use the ResourceSet - // directly. - // TODO(atumanov): implement a ResourceSet const_iterator. - // If light heartbeat enabled, we only set filed that represent resources changed. - if (light_heartbeat_enabled_) { - auto last_heartbeat_resources = gcs_client_->Nodes().GetLastHeartbeatResources(); - if (!last_heartbeat_resources->GetTotalResources().IsEqual( - local_resources.GetTotalResources())) { - for (const auto &resource_pair : - local_resources.GetTotalResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_total())[resource_pair.first] = - resource_pair.second; - } - last_heartbeat_resources->SetTotalResources( - ResourceSet(local_resources.GetTotalResources())); - } - - if (!last_heartbeat_resources->GetAvailableResources().IsEqual( - local_resources.GetAvailableResources())) { - heartbeat_data->set_resources_available_changed(true); - for (const auto &resource_pair : - local_resources.GetAvailableResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_available())[resource_pair.first] = - resource_pair.second; - } - last_heartbeat_resources->SetAvailableResources( - ResourceSet(local_resources.GetAvailableResources())); - } - - local_resources.SetLoadResources(local_queues_.GetTotalResourceLoad()); - if (!last_heartbeat_resources->GetLoadResources().IsEqual( - local_resources.GetLoadResources())) { - heartbeat_data->set_resource_load_changed(true); - for (const auto &resource_pair : - local_resources.GetLoadResources().GetResourceMap()) { - (*heartbeat_data->mutable_resource_load())[resource_pair.first] = - resource_pair.second; - } - last_heartbeat_resources->SetLoadResources( - ResourceSet(local_resources.GetLoadResources())); - } - } else { - // If light heartbeat disabled, we send whole resources information every time. - for (const auto &resource_pair : - local_resources.GetTotalResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_total())[resource_pair.first] = - resource_pair.second; - } - - for (const auto &resource_pair : - local_resources.GetAvailableResources().GetResourceMap()) { - (*heartbeat_data->mutable_resources_available())[resource_pair.first] = - resource_pair.second; - } - - local_resources.SetLoadResources(local_queues_.GetTotalResourceLoad()); - for (const auto &resource_pair : - local_resources.GetLoadResources().GetResourceMap()) { - (*heartbeat_data->mutable_resource_load())[resource_pair.first] = - resource_pair.second; - } - } - } - - if (!new_scheduler_enabled_) { - // Add resource load by shape. This will be used by the new autoscaler. - auto resource_load = local_queues_.GetResourceLoadByShape( - RayConfig::instance().max_resource_shapes_per_load_report()); - heartbeat_data->mutable_resource_load_by_shape()->Swap(&resource_load); - } - - // Set the global gc bit on the outgoing heartbeat message. - if (should_global_gc_) { - heartbeat_data->set_should_global_gc(true); - should_global_gc_ = false; - } - - // Trigger local GC if needed. This throttles the frequency of local GC calls - // to at most once per heartbeat interval. - if (should_local_gc_) { - DoLocalGC(); - should_local_gc_ = false; - } - RAY_CHECK_OK( - gcs_client_->Nodes().AsyncReportHeartbeat(heartbeat_data, /*done*/ nullptr)); + gcs_client_->Nodes().AsyncReportHeartbeat(heartbeat_data, [](Status status) { + if (status.IsDisconnected()) { + RAY_LOG(FATAL) << "This node has beem marked as dead."; + } + })); if (debug_dump_period_ > 0 && static_cast(now_ms - last_debug_dump_at_ms_) > debug_dump_period_) { @@ -523,19 +449,133 @@ void NodeManager::Heartbeat() { }); } +void NodeManager::ReportResourceUsage() { + auto resources_data = std::make_shared(); + SchedulingResources &local_resources = cluster_resource_map_[self_node_id_]; + resources_data->set_node_id(self_node_id_.Binary()); + + if (new_scheduler_enabled_) { + new_resource_scheduler_->FillResourceUsage(light_report_resource_usage_enabled_, + resources_data); + cluster_task_manager_->FillResourceUsage(light_report_resource_usage_enabled_, + resources_data); + } else { + // TODO(atumanov): modify the heartbeat table protocol to use the ResourceSet + // directly. + // TODO(atumanov): implement a ResourceSet const_iterator. + // If light resource usage report enabled, we only set filed that represent resources + // changed. + if (light_report_resource_usage_enabled_) { + auto last_heartbeat_resources = gcs_client_->Nodes().GetLastResourceUsage(); + if (!last_heartbeat_resources->GetTotalResources().IsEqual( + local_resources.GetTotalResources())) { + for (const auto &resource_pair : + local_resources.GetTotalResources().GetResourceMap()) { + (*resources_data->mutable_resources_total())[resource_pair.first] = + resource_pair.second; + } + last_heartbeat_resources->SetTotalResources( + ResourceSet(local_resources.GetTotalResources())); + } + + if (!last_heartbeat_resources->GetAvailableResources().IsEqual( + local_resources.GetAvailableResources())) { + resources_data->set_resources_available_changed(true); + for (const auto &resource_pair : + local_resources.GetAvailableResources().GetResourceMap()) { + (*resources_data->mutable_resources_available())[resource_pair.first] = + resource_pair.second; + } + last_heartbeat_resources->SetAvailableResources( + ResourceSet(local_resources.GetAvailableResources())); + } + + local_resources.SetLoadResources(local_queues_.GetTotalResourceLoad()); + if (!last_heartbeat_resources->GetLoadResources().IsEqual( + local_resources.GetLoadResources())) { + resources_data->set_resource_load_changed(true); + for (const auto &resource_pair : + local_resources.GetLoadResources().GetResourceMap()) { + (*resources_data->mutable_resource_load())[resource_pair.first] = + resource_pair.second; + } + last_heartbeat_resources->SetLoadResources( + ResourceSet(local_resources.GetLoadResources())); + } + } else { + // If light resource usage report disabled, we send whole resources information + // every time. + for (const auto &resource_pair : + local_resources.GetTotalResources().GetResourceMap()) { + (*resources_data->mutable_resources_total())[resource_pair.first] = + resource_pair.second; + } + + for (const auto &resource_pair : + local_resources.GetAvailableResources().GetResourceMap()) { + (*resources_data->mutable_resources_available())[resource_pair.first] = + resource_pair.second; + } + + local_resources.SetLoadResources(local_queues_.GetTotalResourceLoad()); + for (const auto &resource_pair : + local_resources.GetLoadResources().GetResourceMap()) { + (*resources_data->mutable_resource_load())[resource_pair.first] = + resource_pair.second; + } + } + } + + if (!new_scheduler_enabled_) { + // Add resource load by shape. This will be used by the new autoscaler. + auto resource_load = local_queues_.GetResourceLoadByShape( + RayConfig::instance().max_resource_shapes_per_load_report()); + resources_data->mutable_resource_load_by_shape()->Swap(&resource_load); + } + + // Set the global gc bit on the outgoing heartbeat message. + if (should_global_gc_) { + resources_data->set_should_global_gc(true); + should_global_gc_ = false; + } + + // Trigger local GC if needed. This throttles the frequency of local GC calls + // to at most once per heartbeat interval. + auto now = absl::GetCurrentTimeNanos(); + if (should_local_gc_ || now - last_local_gc_ns_ > local_gc_interval_ns_) { + DoLocalGC(); + should_local_gc_ = false; + last_local_gc_ns_ = now; + } + + if (resources_data->resources_total_size() > 0 || + resources_data->resources_available_changed() || + resources_data->resource_load_changed() || resources_data->should_global_gc()) { + RAY_CHECK_OK( + gcs_client_->Nodes().AsyncReportResourceUsage(resources_data, /*done*/ nullptr)); + } + + // Reset the timer. + report_resources_timer_.expires_from_now(report_resources_period_); + report_resources_timer_.async_wait([this](const boost::system::error_code &error) { + RAY_CHECK(!error); + ReportResourceUsage(); + }); +} + void NodeManager::DoLocalGC() { auto all_workers = worker_pool_.GetAllRegisteredWorkers(); for (const auto &driver : worker_pool_.GetAllRegisteredDrivers()) { all_workers.push_back(driver); } - RAY_LOG(WARNING) << "Sending local GC request to " << all_workers.size() - << " workers. It is due to memory pressure on the local node."; + RAY_LOG(INFO) << "Sending Python GC request to " << all_workers.size() + << " local workers to clean up Python cyclic references."; for (const auto &worker : all_workers) { rpc::LocalGCRequest request; worker->rpc_client()->LocalGC( request, [](const ray::Status &status, const rpc::LocalGCReply &r) { if (!status.ok()) { - RAY_LOG(ERROR) << "Failed to send local GC request: " << status.ToString(); + RAY_LOG(DEBUG) << "Failed to send local GC request: " << status.ToString(); } }); } @@ -593,14 +633,7 @@ void NodeManager::HandleReleaseUnusedBundles( } // Return unused bundle resources. - for (auto iter = bundle_spec_map_.begin(); iter != bundle_spec_map_.end();) { - if (0 == in_use_bundles.count(iter->first)) { - ReturnBundleResources(*iter->second); - bundle_spec_map_.erase(iter++); - } else { - iter++; - } - } + placement_group_resource_manager_->ReturnUnusedBundle(in_use_bundles); send_reply_callback(Status::OK(), nullptr, nullptr); } @@ -619,7 +652,7 @@ void NodeManager::WarnResourceDeadlock() { continue; } // Progress is being made, don't warn. - resource_deadlock_warned_ = false; + resource_deadlock_warned_ = 0; return; } @@ -646,13 +679,17 @@ void NodeManager::WarnResourceDeadlock() { } // Push an warning to the driver that a task is blocked trying to acquire resources. - if (any_pending) { + // To avoid spurious triggers, only take action starting with the second time. + // case resource_deadlock_warned_: 0 => first time, don't do anything yet + // case resource_deadlock_warned_: 1 => second time, print a warning + // case resource_deadlock_warned_: >1 => global gc but don't print any warnings + if (any_pending && resource_deadlock_warned_++ > 0) { // Actor references may be caught in cycles, preventing them from being deleted. // Trigger global GC to hopefully free up resource slots. TriggerGlobalGC(); // Suppress duplicates warning messages. - if (resource_deadlock_warned_) { + if (resource_deadlock_warned_ > 2) { return; } @@ -673,7 +710,6 @@ void NodeManager::WarnResourceDeadlock() { "resource_deadlock", error_message.str(), current_time_ms(), exemplar.GetTaskSpecification().JobId()); RAY_CHECK_OK(gcs_client_->Errors().AsyncReportJobError(error_data_ptr, nullptr)); - resource_deadlock_warned_ = true; } } @@ -722,10 +758,11 @@ void NodeManager::NodeAdded(const GcsNodeInfo &node_info) { std::make_pair(node_info.node_manager_address(), node_info.node_manager_port()); // Fetch resource info for the remote node and update cluster resource map. - RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetResources( + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncGetResources( node_id, - [this, node_id](Status status, - const boost::optional &data) { + [this, node_id]( + Status status, + const boost::optional &data) { if (data) { ResourceSet resource_set; for (auto &resource_entry : *data) { @@ -910,47 +947,47 @@ void NodeManager::TryLocalInfeasibleTaskScheduling() { } } -void NodeManager::HeartbeatAdded(const NodeID &node_id, - const HeartbeatTableData &heartbeat_data) { +void NodeManager::ResourceUsageAdded(const NodeID &node_id, + const rpc::ResourcesData &resource_data) { // Locate the node id in remote node table and update available resources based on - // the received heartbeat information. + // the received resource usage information. auto it = cluster_resource_map_.find(node_id); if (it == cluster_resource_map_.end()) { - // Haven't received the node registration for this node yet, skip this heartbeat. - RAY_LOG(INFO) << "[HeartbeatAdded]: received heartbeat from unknown node id " + // Haven't received the node registration for this node yet, skip this message. + RAY_LOG(INFO) << "[ResourceUsageAdded]: received resource usage from unknown node id " << node_id; return; } // Trigger local GC at the next heartbeat interval. - if (heartbeat_data.should_global_gc()) { + if (resource_data.should_global_gc()) { should_local_gc_ = true; } SchedulingResources &remote_resources = it->second; - // If light heartbeat enabled, we update remote resources only when related resources - // map in heartbeat is not empty. - if (light_heartbeat_enabled_) { - if (heartbeat_data.resources_total_size() > 0) { - ResourceSet remote_total(MapFromProtobuf(heartbeat_data.resources_total())); + // If light resource usage report enabled, we update remote resources only when related + // resources map in heartbeat is not empty. + if (light_report_resource_usage_enabled_) { + if (resource_data.resources_total_size() > 0) { + ResourceSet remote_total(MapFromProtobuf(resource_data.resources_total())); remote_resources.SetTotalResources(std::move(remote_total)); } - if (heartbeat_data.resources_available_changed()) { - ResourceSet remote_available(MapFromProtobuf(heartbeat_data.resources_available())); + if (resource_data.resources_available_changed()) { + ResourceSet remote_available(MapFromProtobuf(resource_data.resources_available())); remote_resources.SetAvailableResources(std::move(remote_available)); } - if (heartbeat_data.resource_load_changed()) { - ResourceSet remote_load(MapFromProtobuf(heartbeat_data.resource_load())); + if (resource_data.resource_load_changed()) { + ResourceSet remote_load(MapFromProtobuf(resource_data.resource_load())); // Extract the load information and save it locally. remote_resources.SetLoadResources(std::move(remote_load)); } } else { - // If light heartbeat disabled, we update remote resources every time. - ResourceSet remote_total(MapFromProtobuf(heartbeat_data.resources_total())); + // If light resource usage report disabled, we update remote resources every time. + ResourceSet remote_total(MapFromProtobuf(resource_data.resources_total())); remote_resources.SetTotalResources(std::move(remote_total)); - ResourceSet remote_available(MapFromProtobuf(heartbeat_data.resources_available())); + ResourceSet remote_available(MapFromProtobuf(resource_data.resources_available())); remote_resources.SetAvailableResources(std::move(remote_available)); - ResourceSet remote_load(MapFromProtobuf(heartbeat_data.resource_load())); + ResourceSet remote_load(MapFromProtobuf(resource_data.resource_load())); // Extract the load information and save it locally. remote_resources.SetLoadResources(std::move(remote_load)); } @@ -989,15 +1026,16 @@ void NodeManager::HeartbeatAdded(const NodeID &node_id, } } -void NodeManager::HeartbeatBatchAdded(const HeartbeatBatchTableData &heartbeat_batch) { - // Update load information provided by each heartbeat. - for (const auto &heartbeat_data : heartbeat_batch.batch()) { - const NodeID &node_id = NodeID::FromBinary(heartbeat_data.node_id()); +void NodeManager::ResourceUsageBatchAdded( + const ResourceUsageBatchData &resource_usage_batch) { + // Update load information provided by each message. + for (const auto &resource_usage : resource_usage_batch.batch()) { + const NodeID &node_id = NodeID::FromBinary(resource_usage.node_id()); if (node_id == self_node_id_) { - // Skip heartbeats from self. + // Skip messages from self. continue; } - HeartbeatAdded(node_id, heartbeat_data); + ResourceUsageAdded(node_id, resource_usage); } } @@ -1193,8 +1231,7 @@ void NodeManager::ProcessRegisterClientRequestMessage( std::string worker_ip_address = string_from_flatbuf(*message->ip_address()); // TODO(suquark): Use `WorkerType` in `common.proto` without type converting. rpc::WorkerType worker_type = static_cast(message->worker_type()); - if ((RayConfig::instance().enable_multi_tenancy() && - (worker_type != rpc::WorkerType::SPILL_WORKER && + if (((worker_type != rpc::WorkerType::SPILL_WORKER && worker_type != rpc::WorkerType::RESTORE_WORKER)) || worker_type == rpc::WorkerType::DRIVER) { RAY_CHECK(!job_id.IsNil()); @@ -1627,9 +1664,7 @@ void NodeManager::HandleRequestWorkerLease(const rpc::RequestWorkerLeaseRequest RAY_CHECK_OK(gcs_client_->Tasks().AsyncAdd(data, nullptr)); } - // Prestart optimization is only needed when multi-tenancy is on. - if (RayConfig::instance().enable_multi_tenancy() && - RayConfig::instance().enable_worker_prestart()) { + if (RayConfig::instance().enable_worker_prestart()) { auto task_spec = task.GetTaskSpecification(); worker_pool_.PrestartWorkers(task_spec, request.backlog_size()); } @@ -1712,7 +1747,7 @@ void NodeManager::HandlePrepareBundleResources( auto bundle_spec = BundleSpecification(request.bundle_spec()); RAY_LOG(DEBUG) << "Request to prepare bundle resources is received, " << bundle_spec.DebugString(); - auto prepared = PrepareBundle(cluster_resource_map_, bundle_spec); + auto prepared = placement_group_resource_manager_->PrepareBundle(bundle_spec); reply->set_success(prepared); send_reply_callback(Status::OK(), nullptr, nullptr); // Call task dispatch to assign work to the new group. @@ -1728,7 +1763,7 @@ void NodeManager::HandleCommitBundleResources( auto bundle_spec = BundleSpecification(request.bundle_spec()); RAY_LOG(DEBUG) << "Request to commit bundle resources is received, " << bundle_spec.DebugString(); - CommitBundle(cluster_resource_map_, bundle_spec); + placement_group_resource_manager_->CommitBundle(bundle_spec); send_reply_callback(Status::OK(), nullptr, nullptr); // Call task dispatch to assign work to the new group. @@ -1767,7 +1802,7 @@ void NodeManager::HandleCancelResourceReserve( } // Return bundle resources. - ReturnBundleResources(bundle_spec); + placement_group_resource_manager_->ReturnBundle(bundle_spec); TryLocalInfeasibleTaskScheduling(); DispatchTasks(local_queues_.GetReadyTasksByClass()); @@ -1905,100 +1940,18 @@ void NodeManager::ProcessSetResourceRequest( // Submit to the resource table. This calls the ResourceCreateUpdated or ResourceDeleted // callback, which updates cluster_resource_map_. if (is_deletion) { - RAY_CHECK_OK( - gcs_client_->Nodes().AsyncDeleteResources(node_id, {resource_name}, nullptr)); + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncDeleteResources( + node_id, {resource_name}, nullptr)); } else { std::unordered_map> data_map; auto resource_table_data = std::make_shared(); resource_table_data->set_resource_capacity(capacity); data_map.emplace(resource_name, resource_table_data); - RAY_CHECK_OK(gcs_client_->Nodes().AsyncUpdateResources(node_id, data_map, nullptr)); + RAY_CHECK_OK( + gcs_client_->NodeResources().AsyncUpdateResources(node_id, data_map, nullptr)); } } -bool NodeManager::PrepareBundle( - std::unordered_map &resource_map, - const BundleSpecification &bundle_spec) { - // We will first delete the existing bundle to ensure idempotent. - // The reason why we do this is: after GCS restarts, placement group can be rescheduled - // directly without rolling back the operations performed before the restart. - const auto &bundle_id = bundle_spec.BundleId(); - auto iter = bundle_state_map_.find(bundle_id); - if (iter != bundle_state_map_.end()) { - if (iter->second->state == CommitState::COMMITTED) { - // If the bundle state is already committed, it means that prepare request is just - // stale. - RAY_LOG(INFO) << "Duplicate prepare bundle request, skip it directly."; - return true; - } else { - // If there was a bundle in prepare state, it already locked resources, we will - // return bundle resources. - ReturnBundleResources(bundle_spec); - } - } - - if (resource_map.count(self_node_id_) > 0) { - resource_map[self_node_id_].SetLoadResources(local_queues_.GetTotalResourceLoad()); - } - // Invoke the scheduling policy. - auto reserve_resource_success = - scheduling_policy_.ScheduleBundle(resource_map, self_node_id_, bundle_spec); - - auto bundle_state = std::make_shared(); - if (reserve_resource_success) { - // Register states. - auto it = bundle_state_map_.find(bundle_id); - // Same bundle cannot be rescheduled. - RAY_CHECK(it == bundle_state_map_.end()); - - // Prepare resources. This shouldn't create formatted placement group resources - // because that'll be done at the commit phase. - bundle_state->acquired_resources = - local_available_resources_.Acquire(bundle_spec.GetRequiredResources()); - resource_map[self_node_id_].PrepareBundleResources( - bundle_spec.PlacementGroupId(), bundle_spec.Index(), - bundle_spec.GetRequiredResources()); - - // Register bundle state. - bundle_state->state = CommitState::PREPARED; - bundle_state_map_.emplace(bundle_id, bundle_state); - bundle_spec_map_.emplace( - bundle_id, std::make_shared(bundle_spec.GetMessage())); - } - return bundle_state->acquired_resources.AvailableResources().size() > 0; -} - -void NodeManager::CommitBundle( - std::unordered_map &resource_map, - const BundleSpecification &bundle_spec) { - // TODO(sang): It is currently not idempotent because we don't retry. Make it idempotent - // once retry is implemented. - const auto &bundle_id = bundle_spec.BundleId(); - auto it = bundle_state_map_.find(bundle_id); - // When bundle is committed, it should've been prepared already. - // If GCS call `CommitBundleResources` after `CancelResourceReserve`, we will skip it - // directly. - if (it == bundle_state_map_.end()) { - RAY_LOG(INFO) << "The bundle has been cancelled. Skip it directly. Bundle info is " - << bundle_spec.DebugString(); - return; - } - const auto &bundle_state = it->second; - bundle_state->state = CommitState::COMMITTED; - const auto &acquired_resources = bundle_state->acquired_resources; - for (auto resource : acquired_resources.AvailableResources()) { - local_available_resources_.CommitBundleResourceIds(bundle_spec.PlacementGroupId(), - bundle_spec.Index(), - resource.first, resource.second); - } - - resource_map[self_node_id_].CommitBundleResources(bundle_spec.PlacementGroupId(), - bundle_spec.Index(), - bundle_spec.GetRequiredResources()); - RAY_CHECK(bundle_state->acquired_resources.AvailableResources().size() > 0) - << "Prepare should've been failed if there were no acquireable resources."; -} - void NodeManager::ScheduleTasks( std::unordered_map &resource_map) { // If the resource map contains the local raylet, update load before calling policy. @@ -2228,6 +2181,15 @@ void NodeManager::HandleDirectCallTaskBlocked( void NodeManager::HandleDirectCallTaskUnblocked( const std::shared_ptr &worker) { + if (!worker || worker->GetAssignedTaskId().IsNil()) { + return; // The worker may have died or is no longer processing the task. + } + TaskID task_id = worker->GetAssignedTaskId(); + + // First, always release task dependencies. This ensures we don't leak resources even + // if we don't need to unblock the worker below. + task_dependency_manager_.UnsubscribeGetDependencies(task_id); + if (new_scheduler_enabled_) { // Important: avoid double unblocking if the unblock RPC finishes after task end. if (!worker || !worker->IsBlocked()) { @@ -2247,15 +2209,6 @@ void NodeManager::HandleDirectCallTaskUnblocked( return; } - if (!worker || worker->GetAssignedTaskId().IsNil()) { - return; // The worker may have died or is no longer processing the task. - } - TaskID task_id = worker->GetAssignedTaskId(); - - // First, always release task dependencies. This ensures we don't leak resources even - // if we don't need to unblock the worker below. - task_dependency_manager_.UnsubscribeGetDependencies(task_id); - if (!worker->IsBlocked()) { return; // Don't need to unblock the worker. } @@ -2522,12 +2475,6 @@ bool NodeManager::FinishAssignedTask(const std::shared_ptr &wor task_dependency_manager_.UnsubscribeGetDependencies(spec.TaskId()); task_dependency_manager_.TaskCanceled(task_id); - if (!RayConfig::instance().enable_multi_tenancy()) { - // Unset the worker's assigned job Id if this is not an actor. - if (!spec.IsActorCreationTask()) { - worker.AssignJobId(JobID::Nil()); - } - } if (!spec.IsActorCreationTask()) { // Unset the worker's assigned task. We keep the assigned task ID for // direct actor creation calls because this ID is used later if the actor @@ -2776,9 +2723,7 @@ void NodeManager::FinishAssignTask(const std::shared_ptr &worke // We successfully assigned the task to the worker. worker->AssignTaskId(spec.TaskId()); worker->SetOwnerAddress(spec.CallerAddress()); - if (!RayConfig::instance().enable_multi_tenancy()) { - worker->AssignJobId(spec.JobId()); - } + RAY_CHECK(worker->GetAssignedJobId() == spec.JobId()); // TODO(swang): For actors with multiple actor handles, to // guarantee that tasks are replayed in the same order after a // failure, we must update the task's execution dependency to be @@ -3206,9 +3151,9 @@ void NodeManager::HandleGlobalGC(const rpc::GlobalGCRequest &request, } void NodeManager::TriggerGlobalGC() { - RAY_LOG(WARNING) - << "Broadcasting global GC request to all raylets. This is usually because " - "clusters have memory pressure, and ray needs to GC unused memory."; + RAY_LOG(INFO) << "Broadcasting Python GC request to all raylets since the cluster " + << "is low on resources. This removes Ray actor and object refs " + << "that are stuck in Python reference cycles."; should_global_gc_ = true; // We won't see our own request, so trigger local GC in the next heartbeat. should_local_gc_ = true; @@ -3253,31 +3198,6 @@ void NodeManager::RecordMetrics() { local_queues_.RecordMetrics(); } -bool NodeManager::ReturnBundleResources(const BundleSpecification &bundle_spec) { - // We should commit resources if it weren't because - // ReturnBundleResources requires resources to be committed when it is called. - auto it = bundle_state_map_.find(bundle_spec.BundleId()); - if (it == bundle_state_map_.end()) { - RAY_LOG(INFO) << "Duplicate cancel request, skip it directly."; - return false; - } - const auto &bundle_state = it->second; - if (bundle_state->state == CommitState::PREPARED) { - CommitBundle(cluster_resource_map_, bundle_spec); - } - bundle_state_map_.erase(it); - - // Return resources. - const auto &resource_set = bundle_spec.GetRequiredResources(); - for (const auto &resource : resource_set.GetResourceMap()) { - local_available_resources_.ReturnBundleResources(bundle_spec.PlacementGroupId(), - bundle_spec.Index(), resource.first); - } - cluster_resource_map_[self_node_id_].ReturnBundleResources( - bundle_spec.PlacementGroupId(), bundle_spec.Index()); - return true; -} - } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index b73eb6876..f1e9ffad0 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -40,6 +40,7 @@ #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/ordered_set.h" #include "ray/common/bundle_spec.h" +#include "ray/raylet/placement_group_resource_manager.h" // clang-format on namespace ray { @@ -49,9 +50,9 @@ namespace raylet { using rpc::ActorTableData; using rpc::ErrorType; using rpc::GcsNodeInfo; -using rpc::HeartbeatBatchTableData; using rpc::HeartbeatTableData; using rpc::JobTableData; +using rpc::ResourceUsageBatchData; struct NodeManagerConfig { /// The node's resource configuration. @@ -70,8 +71,6 @@ struct NodeManagerConfig { /// An explicit list of open ports that workers started will bind /// on. This takes precedence over min_worker_port and max_worker_port. std::vector worker_ports; - /// The initial number of workers to create. - int num_initial_workers; /// The soft limit of the number of workers. int num_workers_soft_limit; /// Number of initial Python workers for the first job. @@ -85,6 +84,8 @@ struct NodeManagerConfig { std::string agent_command; /// The time between heartbeats in milliseconds. uint64_t heartbeat_period_ms; + /// The time between reports resources in milliseconds. + uint64_t report_resources_period_ms; /// The time between debug dumps in milliseconds, or -1 to disable. uint64_t debug_dump_period_ms; /// Whether to enable fair queueing between task classes in raylet. @@ -105,27 +106,6 @@ struct NodeManagerConfig { uint64_t record_metrics_period_ms; }; -struct pair_hash { - template - std::size_t operator()(const std::pair &pair) const { - return std::hash()(pair.first) ^ std::hash()(pair.second); - } -}; - -enum CommitState { - /// Resources are prepared. - PREPARED, - /// Resources are COMMITTED. - COMMITTED -}; - -struct BundleState { - /// Leasing state for 2PC protocol. - CommitState state; - /// Resources that are acquired at preparation stage. - ResourceIdSet acquired_resources; -}; - class NodeManager : public rpc::NodeManagerServiceHandler { public: /// Create a node manager. @@ -220,6 +200,9 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// Send heartbeats to the GCS. void Heartbeat(); + /// Report resource usage to the GCS. + void ReportResourceUsage(); + /// Write out debug state to a file. void DumpDebugState() const; @@ -232,16 +215,16 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// \return Void. void GetObjectManagerProfileInfo(); - /// Handler for a heartbeat notification from the GCS. + /// Handler for a resource usage notification from the GCS. /// - /// \param id The ID of the node manager that sent the heartbeat. - /// \param data The heartbeat data including load information. + /// \param id The ID of the node manager that sent the resources data. + /// \param data The resources data including load information. /// \return Void. - void HeartbeatAdded(const NodeID &id, const HeartbeatTableData &data); - /// Handler for a heartbeat batch notification from the GCS + void ResourceUsageAdded(const NodeID &id, const rpc::ResourcesData &data); + /// Handler for a resource usage batch notification from the GCS /// - /// \param heartbeat_batch The batch of heartbeat data. - void HeartbeatBatchAdded(const HeartbeatBatchTableData &heartbeat_batch); + /// \param resource_usage_batch The batch of resource usage data. + void ResourceUsageBatchAdded(const ResourceUsageBatchData &resource_usage_batch); /// Methods for task scheduling. @@ -682,14 +665,19 @@ class NodeManager : public rpc::NodeManagerServiceHandler { boost::asio::steady_timer heartbeat_timer_; /// The period used for the heartbeat timer. std::chrono::milliseconds heartbeat_period_; + /// The timer used to report resources. + boost::asio::steady_timer report_resources_timer_; + /// The period used for the resources report timer. + std::chrono::milliseconds report_resources_period_; /// The period between debug state dumps. int64_t debug_dump_period_; /// Whether to enable fair queueing between task classes in raylet. bool fair_queueing_enabled_; /// Whether to enable pinning for plasma objects. bool object_pinning_enabled_; - /// Whether we have printed out a resource deadlock warning. - bool resource_deadlock_warned_ = false; + /// Incremented each time we encounter a potential resource deadlock condition. + /// This is reset to zero when the condition is cleared. + int resource_deadlock_warned_ = 0; /// Whether we have recorded any metrics yet. bool recorded_metrics_ = false; /// The path to the ray temp dir. @@ -700,8 +688,8 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// The time that the last heartbeat was sent at. Used to make sure we are /// keeping up with heartbeats. uint64_t last_heartbeat_at_ms_; - /// Only the changed part will be included in heartbeat if this is true. - const bool light_heartbeat_enabled_; + /// Only the changed part will be included in resource usage if this is true. + const bool light_report_resource_usage_enabled_; /// The time that the last debug string was logged to the console. uint64_t last_debug_dump_at_ms_; /// The number of heartbeats that we should wait before sending the @@ -710,6 +698,7 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// Initial node manager configuration. const NodeManagerConfig initial_config_; /// The resources (and specific resource IDs) that are currently available. + /// These two resource container is shared with `PlacementGroupResourceManager`. ResourceIdSet local_available_resources_; std::unordered_map cluster_resource_map_; @@ -772,6 +761,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// on all local workers of this raylet. bool should_local_gc_ = false; + /// The last time local GC was triggered. + int64_t last_local_gc_ns_ = 0; + + /// The interval in nanoseconds between local GC automatic triggers. + const int64_t local_gc_interval_ns_ = 10 * 60 * 1e9; + /// These two classes make up the new scheduler. ClusterResourceScheduler is /// responsible for maintaining a view of the cluster state w.r.t resource /// usage. ClusterTaskManager is responsible for queuing, spilling back, and @@ -794,15 +789,6 @@ class NodeManager : public rpc::NodeManagerServiceHandler { absl::flat_hash_map>> async_plasma_objects_notification_ GUARDED_BY(plasma_object_notification_lock_); - /// This map represents the commit state of 2PC protocol for atomic placement group - /// creation. - absl::flat_hash_map, pair_hash> - bundle_state_map_; - - /// Save `BundleSpecification` for cleaning leaked bundles after GCS restart. - absl::flat_hash_map, pair_hash> - bundle_spec_map_; - /// Fields that are used to report metrics. /// The period between debug state dumps. int64_t record_metrics_period_; @@ -818,6 +804,9 @@ class NodeManager : public rpc::NodeManagerServiceHandler { /// Number of tasks that are spilled back to other nodes. uint64_t metrics_num_task_spilled_back_; + + /// Managers all bundle-related operations. + std::shared_ptr placement_group_resource_manager_; }; } // namespace raylet diff --git a/src/ray/raylet/object_manager_integration_test.cc b/src/ray/raylet/object_manager_integration_test.cc index 5f950f46e..0a1aae200 100644 --- a/src/ray/raylet/object_manager_integration_test.cc +++ b/src/ray/raylet/object_manager_integration_test.cc @@ -40,7 +40,6 @@ class TestObjectManagerBase : public ::testing::Test { static_resource_conf = {{"CPU", 1}, {"GPU", 1}}; node_manager_config.resource_config = ray::raylet::ResourceSet(std::move(static_resource_conf)); - node_manager_config.num_initial_workers = 0; // Use a default worker that can execute empty tasks with dependencies. std::vector py_worker_command; py_worker_command.push_back("python"); diff --git a/src/ray/raylet/placement_group_resource_manager.cc b/src/ray/raylet/placement_group_resource_manager.cc new file mode 100644 index 000000000..06a82663e --- /dev/null +++ b/src/ray/raylet/placement_group_resource_manager.cc @@ -0,0 +1,152 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +#include "ray/raylet/placement_group_resource_manager.h" + +#include +#include +#include + +namespace ray { + +namespace raylet { + +OldPlacementGroupResourceManager::OldPlacementGroupResourceManager( + ResourceIdSet &local_available_resources_, + std::unordered_map &cluster_resource_map_, + const NodeID &self_node_id_) + : local_available_resources_(local_available_resources_), + cluster_resource_map_(cluster_resource_map_), + self_node_id_(self_node_id_) {} + +bool OldPlacementGroupResourceManager::PrepareBundle( + const BundleSpecification &bundle_spec) { + // We will first delete the existing bundle to ensure idempotence. + // The reason why we do this is: after GCS restarts, placement group can be rescheduled + // directly without rolling back the operations performed before the restart. + const auto &bundle_id = bundle_spec.BundleId(); + auto iter = bundle_state_map_.find(bundle_id); + if (iter != bundle_state_map_.end()) { + if (iter->second->state == CommitState::COMMITTED) { + // If the bundle state is already committed, it means that prepare request is just + // stale. + RAY_LOG(INFO) << "Duplicate prepare bundle request, skip it directly."; + return true; + } else { + // If there was a bundle in prepare state, it already locked resources, we will + // return bundle resources. + ReturnBundle(bundle_spec); + } + } + + auto &local_resource_set = cluster_resource_map_[self_node_id_]; + auto bundle_state = std::make_shared(); + bool local_resource_enough = bundle_spec.GetRequiredResources().IsSubset( + local_resource_set.GetAvailableResources()); + + if (local_resource_enough) { + // Register states. + auto it = bundle_state_map_.find(bundle_id); + // Same bundle cannot be rescheduled. + RAY_CHECK(it == bundle_state_map_.end()); + + // Prepare resources. This shouldn't create formatted placement group resources + // because that'll be done at the commit phase. + bundle_state->acquired_resources = + local_available_resources_.Acquire(bundle_spec.GetRequiredResources()); + local_resource_set.Acquire(bundle_spec.GetRequiredResources()); + + // Register bundle state. + bundle_state->state = CommitState::PREPARED; + bundle_state_map_.emplace(bundle_id, bundle_state); + bundle_spec_map_.emplace( + bundle_id, std::make_shared(bundle_spec.GetMessage())); + } + return bundle_state->acquired_resources.AvailableResources().size() > 0; +} + +void OldPlacementGroupResourceManager::CommitBundle( + const BundleSpecification &bundle_spec) { + const auto &bundle_id = bundle_spec.BundleId(); + auto it = bundle_state_map_.find(bundle_id); + // When bundle is committed, it should've been prepared already. + // If GCS call `CommitBundleResources` after `CancelResourceReserve`, we will skip it + // directly. + if (it == bundle_state_map_.end()) { + RAY_LOG(INFO) << "The bundle has been cancelled. Skip it directly. Bundle info is " + << bundle_spec.DebugString(); + return; + } else { + // Ignore request If the bundle state is already committed. + if (it->second->state == CommitState::COMMITTED) { + RAY_LOG(INFO) << "Duplicate committ bundle request, skip it directly."; + return; + } + } + const auto &bundle_state = it->second; + bundle_state->state = CommitState::COMMITTED; + const auto &acquired_resources = bundle_state->acquired_resources; + + const auto &bundle_resource_labels = bundle_spec.GetFormattedResources(); + const auto &formatted_resource_set = ResourceSet(bundle_resource_labels); + local_available_resources_.Release(ResourceIdSet(formatted_resource_set)); + + cluster_resource_map_[self_node_id_].AddResource(ResourceSet(bundle_resource_labels)); + + RAY_CHECK(acquired_resources.AvailableResources().size() > 0) + << "Prepare should've been failed if there were no acquireable resources."; +} + +void OldPlacementGroupResourceManager::ReturnBundle( + const BundleSpecification &bundle_spec) { + // We should commit resources if it weren't because + // ReturnBundleResources requires resources to be committed when it is called. + auto it = bundle_state_map_.find(bundle_spec.BundleId()); + if (it == bundle_state_map_.end()) { + RAY_LOG(INFO) << "Duplicate cancel request, skip it directly."; + return; + } + const auto &bundle_state = it->second; + if (bundle_state->state == CommitState::PREPARED) { + CommitBundle(bundle_spec); + } + bundle_state_map_.erase(it); + + const auto &resource_set = bundle_spec.GetRequiredResources(); + const auto &placement_group_resource_labels = bundle_spec.GetFormattedResources(); + + // Return resources to ResourceIdSet. + local_available_resources_.Release(ResourceIdSet(resource_set)); + local_available_resources_.Acquire(ResourceSet(placement_group_resource_labels)); + + // Return resources to SchedulingResources. + cluster_resource_map_[self_node_id_].Release(resource_set); + cluster_resource_map_[self_node_id_].Acquire( + ResourceSet(placement_group_resource_labels)); +} + +void OldPlacementGroupResourceManager::ReturnUnusedBundle( + const std::unordered_set &in_use_bundles) { + for (auto iter = bundle_spec_map_.begin(); iter != bundle_spec_map_.end();) { + if (0 == in_use_bundles.count(iter->first)) { + ReturnBundle(*iter->second); + bundle_spec_map_.erase(iter++); + } else { + iter++; + } + } +} + +} // namespace raylet +} // namespace ray diff --git a/src/ray/raylet/placement_group_resource_manager.h b/src/ray/raylet/placement_group_resource_manager.h new file mode 100644 index 000000000..3b6e3a928 --- /dev/null +++ b/src/ray/raylet/placement_group_resource_manager.h @@ -0,0 +1,131 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/container/flat_hash_map.h" +#include "ray/common/bundle_spec.h" +#include "ray/common/id.h" +#include "ray/common/task/scheduling_resources.h" + +namespace ray { + +namespace raylet { + +enum CommitState { + /// Resources are prepared. + PREPARED, + /// Resources are COMMITTED. + COMMITTED +}; + +struct BundleState { + /// Leasing state for 2PC protocol. + CommitState state; + /// Resources that are acquired at preparation stage. + ResourceIdSet acquired_resources; +}; + +struct pair_hash { + template + std::size_t operator()(const std::pair &pair) const { + return std::hash()(pair.first) ^ std::hash()(pair.second); + } +}; + +/// `PlacementGroupResourceManager` responsible for managing the resources that +/// about allocated for placement group bundles. +class PlacementGroupResourceManager { + public: + /// Lock the required resources from local available resources. Note that this is phase + /// one of 2PC, it will not convert placement group resource(like CPU -> CPU_group_i). + /// + /// \param bundle_spec: Specification of bundle whose resources will be prepared. + virtual bool PrepareBundle(const BundleSpecification &bundle_spec) = 0; + + /// Convert the required resources to placement group resources(like CPU -> + /// CPU_group_i). This is phase two of 2PC. + /// + /// \param bundle_spec: Specification of bundle whose resources will be commited. + virtual void CommitBundle(const BundleSpecification &bundle_spec) = 0; + + /// Return back all the bundle resource. + /// + /// \param bundle_spec: Specification of bundle whose resources will be returned. + virtual void ReturnBundle(const BundleSpecification &bundle_spec) = 0; + + /// Return back all the bundle(which is unused) resource. + /// + /// \param bundle_spec: A set of bundles which in use. + virtual void ReturnUnusedBundle( + const std::unordered_set &in_use_bundles) = 0; + + virtual ~PlacementGroupResourceManager() {} +}; + +/// Associated with old scheduler. +class OldPlacementGroupResourceManager : public PlacementGroupResourceManager { + public: + /// Create a local placement group manager. + /// + /// \param local_available_resources_: The resources (IDs specificed) that are currently + /// available. + /// \param cluster_resource_map_: The resources (without IDs specificed) that + /// are currently available. + /// \param self_node_id_: The related raylet with current + /// placement group manager. + OldPlacementGroupResourceManager( + ResourceIdSet &local_available_resources_, + std::unordered_map &cluster_resource_map_, + const NodeID &self_node_id_); + + virtual ~OldPlacementGroupResourceManager() = default; + + bool PrepareBundle(const BundleSpecification &bundle_spec); + + void CommitBundle(const BundleSpecification &bundle_spec); + + void ReturnBundle(const BundleSpecification &bundle_spec); + + void ReturnUnusedBundle(const std::unordered_set &in_use_bundles); + + /// Get all local available resource(IDs specificed). + const ResourceIdSet &GetAllResourceIdSet() const { return local_available_resources_; }; + + /// Get all local available resource(without IDs specificed). + const SchedulingResources &GetAllResourceSetWithoutId() const { + return cluster_resource_map_[self_node_id_]; + } + + private: + /// The resources (and specific resource IDs) that are currently available. + /// These two resource container is shared with `NodeManager`. + ResourceIdSet &local_available_resources_; + std::unordered_map &cluster_resource_map_; + + /// Related raylet with current placement group manager. + NodeID self_node_id_; + + /// This map represents the commit state of 2PC protocol for atomic placement group + /// creation. + absl::flat_hash_map, pair_hash> + bundle_state_map_; + + /// Save `BundleSpecification` for cleaning leaked bundles after GCS restart. + absl::flat_hash_map, pair_hash> + bundle_spec_map_; +}; + +} // namespace raylet +} // end namespace ray diff --git a/src/ray/raylet/placement_group_resource_manager_test.cc b/src/ray/raylet/placement_group_resource_manager_test.cc new file mode 100644 index 000000000..10011aece --- /dev/null +++ b/src/ray/raylet/placement_group_resource_manager_test.cc @@ -0,0 +1,270 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/raylet/placement_group_resource_manager.h" +#include "ray/common/bundle_spec.h" +#include "ray/common/id.h" +#include "ray/common/task/scheduling_resources.h" +#include "ray/gcs/test/gcs_test_util.h" + +#include + +#include "gtest/gtest.h" + +namespace ray { + +class OldPlacementGroupResourceManagerTest : public ::testing::Test { + public: + OldPlacementGroupResourceManagerTest() { + old_placement_group_resource_manager_.reset( + new raylet::OldPlacementGroupResourceManager( + local_available_resources_, cluster_resource_map_, self_node_id_)); + } + + std::unique_ptr + old_placement_group_resource_manager_; + + void InitLocalAvailableResource( + std::unordered_map &unit_resource) { + ResourceSet init_resourece(unit_resource); + cluster_resource_map_[self_node_id_] = SchedulingResources(init_resourece); + local_available_resources_ = ResourceIdSet(init_resourece); + } + + void CheckRemainingResourceCorrect(ResourceSet &result_resource) { + auto &remaining_resource = + old_placement_group_resource_manager_->GetAllResourceSetWithoutId(); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); + } + + protected: + ResourceIdSet local_available_resources_; + std::unordered_map cluster_resource_map_; + NodeID self_node_id_ = NodeID::FromRandom(); +}; + +TEST_F(OldPlacementGroupResourceManagerTest, TestPrepareBundleResource) { + // 1. create bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + InitLocalAvailableResource(unit_resource); + /// 3. prepare bundle resource. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + /// 4. check remaining resources is correct. + auto &remaining_resource = + old_placement_group_resource_manager_->GetAllResourceSetWithoutId(); + ResourceSet result_resource; + ASSERT_EQ(0, local_available_resources_.AvailableResources().size()); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestPrepareBundleWithInsufficientResource) { + // 1. create bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 2.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + std::unordered_map init_unit_resource; + init_unit_resource.insert({"CPU", 1.0}); + InitLocalAvailableResource(init_unit_resource); + /// 3. prepare bundle resource. + ASSERT_FALSE(old_placement_group_resource_manager_->PrepareBundle(bundle_spec)); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestCommitBundleResource) { + // 1. create bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + InitLocalAvailableResource(unit_resource); + /// 3. prepare and commit bundle resource. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + /// 4. check remaining resources is correct. + auto &remaining_resource = + old_placement_group_resource_manager_->GetAllResourceSetWithoutId(); + std::vector resource_labels = {"CPU_group_" + group_id.Hex(), + "CPU_group_1_" + group_id.Hex()}; + std::vector resource_capacity = {1.0, 1.0}; + ResourceSet result_resource(resource_labels, resource_capacity); + ASSERT_EQ(2, local_available_resources_.AvailableResources().size()); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestReturnBundleResource) { + // 1. create bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + InitLocalAvailableResource(unit_resource); + /// 3. prepare and commit bundle resource. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + /// 4. return bundle resource. + old_placement_group_resource_manager_->ReturnBundle(bundle_spec); + /// 5. check remaining resources is correct. + auto &remaining_resource = + old_placement_group_resource_manager_->GetAllResourceSetWithoutId(); + ResourceSet result_resource(unit_resource); + ASSERT_EQ(1, local_available_resources_.AvailableResources().size()); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestMultipleBundlesCommitAndReturn) { + // 1. create two bundles spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto first_bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + auto second_bundle_spec = Mocker::GenBundleCreation(group_id, 2, unit_resource); + /// 2. init local available resource. + std::unordered_map init_unit_resource; + init_unit_resource.insert({"CPU", 2.0}); + InitLocalAvailableResource(init_unit_resource); + /// 3. prepare and commit two bundle resource. + old_placement_group_resource_manager_->PrepareBundle(first_bundle_spec); + old_placement_group_resource_manager_->PrepareBundle(second_bundle_spec); + old_placement_group_resource_manager_->CommitBundle(first_bundle_spec); + old_placement_group_resource_manager_->CommitBundle(second_bundle_spec); + /// 4. check remaining resources is correct after commit phase. + auto &remaining_resource = + old_placement_group_resource_manager_->GetAllResourceSetWithoutId(); + std::vector resource_labels = {"CPU_group_" + group_id.Hex(), + "CPU_group_1_" + group_id.Hex(), + "CPU_group_2_" + group_id.Hex()}; + std::vector resource_capacity = {2.0, 1.0, 1.0}; + ResourceSet result_resource(resource_labels, resource_capacity); + ASSERT_EQ(3, local_available_resources_.AvailableResources().size()); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); + /// 5. return second bundle. + old_placement_group_resource_manager_->ReturnBundle(second_bundle_spec); + /// 6. check remaining resources is correct after return second bundle. + resource_labels = {"CPU", "CPU_group_" + group_id.Hex(), + "CPU_group_1_" + group_id.Hex()}; + resource_capacity = {1.0, 1.0, 1.0}; + result_resource = ResourceSet(resource_labels, resource_capacity); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); + /// 7. return first bundel. + old_placement_group_resource_manager_->ReturnBundle(first_bundle_spec); + /// 8. check remaining resources is correct after all bundle returned. + result_resource = ResourceSet(init_unit_resource); + ASSERT_EQ(1, remaining_resource.GetAvailableResources().IsEqual(result_resource)) + << remaining_resource.GetAvailableResources().ToString() << " vs " + << result_resource.ToString(); + ASSERT_EQ(1, local_available_resources_.ToResourceSet().IsEqual(result_resource)) + << local_available_resources_.ToResourceSet().ToString() << " vs " + << result_resource.ToString(); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestIdempotencyWithMultiPrepare) { + // 1. create one bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + std::unordered_map available_resource = { + std::make_pair("CPU", 3.0)}; + InitLocalAvailableResource(available_resource); + /// 3. prepare bundle resource 10 times. + for (int i = 0; i < 10; i++) { + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + } + /// 4. check remaining resources is correct. + std::unordered_map result_resource_map = { + std::make_pair("CPU", 2.0)}; + ResourceSet result_resource(result_resource_map); + CheckRemainingResourceCorrect(result_resource); +} + +TEST_F(OldPlacementGroupResourceManagerTest, TestIdempotencyWithRandomOrder) { + // 1. create one bundle spec. + auto group_id = PlacementGroupID::FromRandom(); + std::unordered_map unit_resource; + unit_resource.insert({"CPU", 1.0}); + auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + /// 2. init local available resource. + std::unordered_map available_resource = { + std::make_pair("CPU", 3.0)}; + InitLocalAvailableResource(available_resource); + /// 3. prepare bundle -> commit bundle -> prepare bundle. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + /// 4. check remaining resources is correct. + std::vector resource_labels = {"CPU_group_" + group_id.Hex(), + "CPU_group_1_" + group_id.Hex(), "CPU"}; + std::vector resource_capacity = {1.0, 1.0, 2.0}; + ResourceSet result_resource(resource_labels, resource_capacity); + CheckRemainingResourceCorrect(result_resource); + old_placement_group_resource_manager_->ReturnBundle(bundle_spec); + // 5. prepare bundle -> commit bundle -> commit bundle. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + // 6. check remaining resources is correct. + CheckRemainingResourceCorrect(result_resource); + old_placement_group_resource_manager_->ReturnBundle(bundle_spec); + // 7. prepare bundle -> return bundle -> commit bundle. + old_placement_group_resource_manager_->PrepareBundle(bundle_spec); + old_placement_group_resource_manager_->ReturnBundle(bundle_spec); + old_placement_group_resource_manager_->CommitBundle(bundle_spec); + result_resource = ResourceSet(available_resource); + CheckRemainingResourceCorrect(result_resource); +} + +} // namespace ray + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/ray/raylet/raylet.cc b/src/ray/raylet/raylet.cc index 26e03b12e..d2ddead62 100644 --- a/src/ray/raylet/raylet.cc +++ b/src/ray/raylet/raylet.cc @@ -136,8 +136,8 @@ ray::Status Raylet::RegisterGcs() { resource->set_resource_capacity(resource_pair.second); resources.emplace(resource_pair.first, resource); } - RAY_CHECK_OK( - gcs_client_->Nodes().AsyncUpdateResources(self_node_id_, resources, nullptr)); + RAY_CHECK_OK(gcs_client_->NodeResources().AsyncUpdateResources(self_node_id_, + resources, nullptr)); RAY_CHECK_OK(node_manager_.RegisterGcs()); }; diff --git a/src/ray/raylet/scheduling/cluster_resource_data.cc b/src/ray/raylet/scheduling/cluster_resource_data.cc index 91856c5b8..cb0214dab 100644 --- a/src/ray/raylet/scheduling/cluster_resource_data.cc +++ b/src/ray/raylet/scheduling/cluster_resource_data.cc @@ -83,7 +83,8 @@ TaskRequest ResourceMapToTaskRequest( } else if (resource.first == ray::kMemory_ResourceLabel) { task_request.predefined_resources[MEM].demand = resource.second; } else { - task_request.custom_resources[i].id = string_to_int_map.Insert(resource.first); + string_to_int_map.Insert(resource.first); + task_request.custom_resources[i].id = string_to_int_map.Get(resource.first); task_request.custom_resources[i].demand = resource.second; task_request.custom_resources[i].soft = false; i++; @@ -195,6 +196,8 @@ bool NodeResources::operator==(const NodeResources &other) { return true; } +bool NodeResources::operator!=(const NodeResources &other) { return !(*this == other); } + std::string NodeResources::DebugString(StringIdMap string_to_in_map) const { std::stringstream buffer; buffer << " {\n"; diff --git a/src/ray/raylet/scheduling/cluster_resource_data.h b/src/ray/raylet/scheduling/cluster_resource_data.h index 9c769ecf4..96b4c4359 100644 --- a/src/ray/raylet/scheduling/cluster_resource_data.h +++ b/src/ray/raylet/scheduling/cluster_resource_data.h @@ -161,6 +161,7 @@ class NodeResources { absl::flat_hash_map custom_resources; /// Returns if this equals another node resources. bool operator==(const NodeResources &other); + bool operator!=(const NodeResources &other); /// Returns human-readable string for these resources. std::string DebugString(StringIdMap string_to_int_map) const; }; diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler.cc b/src/ray/raylet/scheduling/cluster_resource_scheduler.cc index 8154a858f..2245760c2 100644 --- a/src/ray/raylet/scheduling/cluster_resource_scheduler.cc +++ b/src/ray/raylet/scheduling/cluster_resource_scheduler.cc @@ -43,25 +43,6 @@ void ClusterResourceScheduler::AddOrUpdateNode( AddOrUpdateNode(string_to_int_map_.Insert(node_id), node_resources); } -void ClusterResourceScheduler::SetPredefinedResources(const NodeResources &new_resources, - NodeResources *old_resources) { - for (size_t i = 0; i < PredefinedResources_MAX; i++) { - old_resources->predefined_resources[i].total = - new_resources.predefined_resources[i].total; - old_resources->predefined_resources[i].available = - new_resources.predefined_resources[i].available; - } -} - -void ClusterResourceScheduler::SetCustomResources( - const absl::flat_hash_map &new_custom_resources, - absl::flat_hash_map *old_custom_resources) { - old_custom_resources->clear(); - for (auto &elem : new_custom_resources) { - old_custom_resources->insert(elem); - } -} - void ClusterResourceScheduler::AddOrUpdateNode(int64_t node_id, const NodeResources &node_resources) { auto it = nodes_.find(node_id); @@ -70,9 +51,7 @@ void ClusterResourceScheduler::AddOrUpdateNode(int64_t node_id, nodes_.emplace(node_id, node_resources); } else { // This node exists, so update its resources. - NodeResources &resources = it->second; - SetPredefinedResources(node_resources, &resources); - SetCustomResources(node_resources.custom_resources, &resources.custom_resources); + it->second = Node(node_resources); } } @@ -82,7 +61,6 @@ bool ClusterResourceScheduler::RemoveNode(int64_t node_id) { // Node not found. return false; } else { - it->second.custom_resources.clear(); nodes_.erase(it); string_to_int_map_.Remove(node_id); return true; @@ -98,9 +76,43 @@ bool ClusterResourceScheduler::RemoveNode(const std::string &node_id_string) { return RemoveNode(node_id); } +bool ClusterResourceScheduler::IsLocallyFeasible( + const std::unordered_map shape) { + const TaskRequest task_req = ResourceMapToTaskRequest(string_to_int_map_, shape); + RAY_CHECK(nodes_.contains(local_node_id_)); + const auto &it = nodes_.find(local_node_id_); + RAY_CHECK(it != nodes_.end()); + return IsFeasible(task_req, it->second.GetLocalView()); +} + +bool ClusterResourceScheduler::IsFeasible(const TaskRequest &task_req, + const NodeResources &resources) const { + // First, check predefined resources. + for (size_t i = 0; i < PredefinedResources_MAX; i++) { + if (task_req.predefined_resources[i].demand > + resources.predefined_resources[i].total) { + return false; + } + } + + // Now check custom resources. + for (const auto &task_req_custom_resource : task_req.custom_resources) { + auto it = resources.custom_resources.find(task_req_custom_resource.id); + + if (it == resources.custom_resources.end()) { + return false; + } + if (task_req_custom_resource.demand > it->second.total) { + return false; + } + } + + return true; +} + int64_t ClusterResourceScheduler::IsSchedulable(const TaskRequest &task_req, int64_t node_id, - const NodeResources &resources) { + const NodeResources &resources) const { int violations = 0; // First, check predefined resources. @@ -189,9 +201,10 @@ int64_t ClusterResourceScheduler::GetBestSchedulableNode(const TaskRequest &task // Check whether local node is schedulable. We return immediately // the local node only if there are zero violations. - auto it = nodes_.find(local_node_id_); - if (it != nodes_.end()) { - if (IsSchedulable(task_req, it->first, it->second) == 0) { + const auto local_node_it = nodes_.find(local_node_id_); + if (local_node_it != nodes_.end()) { + if (IsSchedulable(task_req, local_node_it->first, + local_node_it->second.GetLocalView()) == 0) { return local_node_id_; } } @@ -201,18 +214,31 @@ int64_t ClusterResourceScheduler::GetBestSchedulableNode(const TaskRequest &task for (const auto &task_req_placement_hint : task_req.placement_hints) { auto it = nodes_.find(task_req_placement_hint); if (it != nodes_.end()) { - if (IsSchedulable(task_req, it->first, it->second) == 0) { + if (IsSchedulable(task_req, it->first, it->second.GetLocalView()) == 0) { return it->first; } } } + bool local_node_feasible = IsFeasible(task_req, local_node_it->second.GetLocalView()); + for (const auto &node : nodes_) { // Return -1 if node not schedulable. otherwise return the number // of soft constraint violations. - int64_t violations; - - if ((violations = IsSchedulable(task_req, node.first, node.second)) == -1) { + int64_t violations = IsSchedulable(task_req, node.first, node.second.GetLocalView()); + if (violations == -1) { + if (!local_node_feasible && best_node == -1 && + IsFeasible(task_req, node.second.GetLocalView())) { + // If the local node is not feasible, and a better node has not yet + // been found, and this node does not currently have the resources + // available but is feasible, then schedule to this node. + // NOTE(swang): This is needed to make sure that tasks that are not + // feasible on this node are spilled back to a node that does have the + // appropriate total resources in a timely manner. If there are + // multiple feasible nodes, this algorithm can still introduce delays + // because of inefficient load-balancing. + best_node = node.first; + } continue; } @@ -246,30 +272,32 @@ std::string ClusterResourceScheduler::GetBestSchedulableNode( return string_to_int_map_.Get(node_id); } -bool ClusterResourceScheduler::SubtractNodeAvailableResources( +bool ClusterResourceScheduler::SubtractRemoteNodeAvailableResources( int64_t node_id, const TaskRequest &task_req) { + RAY_CHECK(node_id != local_node_id_); + auto it = nodes_.find(node_id); if (it == nodes_.end()) { return false; } - NodeResources &resources = it->second; + NodeResources *resources = it->second.GetMutableLocalView(); // Just double check this node can still schedule the task request. - if (IsSchedulable(task_req, node_id, resources) == -1) { + if (IsSchedulable(task_req, node_id, *resources) == -1) { return false; } FixedPoint zero(0.); for (size_t i = 0; i < PredefinedResources_MAX; i++) { - resources.predefined_resources[i].available = - std::max(FixedPoint(0), resources.predefined_resources[i].available - + resources->predefined_resources[i].available = + std::max(FixedPoint(0), resources->predefined_resources[i].available - task_req.predefined_resources[i].demand); } for (const auto &task_req_custom_resource : task_req.custom_resources) { - auto it = resources.custom_resources.find(task_req_custom_resource.id); - if (it != resources.custom_resources.end()) { + auto it = resources->custom_resources.find(task_req_custom_resource.id); + if (it != resources->custom_resources.end()) { it->second.available = std::max(FixedPoint(0), it->second.available - task_req_custom_resource.demand); } @@ -277,50 +305,11 @@ bool ClusterResourceScheduler::SubtractNodeAvailableResources( return true; } -bool ClusterResourceScheduler::SubtractNodeAvailableResources( - const std::string &node_id, - const std::unordered_map &resource_map) { - TaskRequest task_request = ResourceMapToTaskRequest(string_to_int_map_, resource_map); - return SubtractNodeAvailableResources(string_to_int_map_.Get(node_id), task_request); -} - -bool ClusterResourceScheduler::AddNodeAvailableResources(int64_t node_id, - const TaskRequest &task_req) { - auto it = nodes_.find(node_id); - if (it == nodes_.end()) { - return false; - } - NodeResources &resources = it->second; - - for (size_t i = 0; i < PredefinedResources_MAX; i++) { - resources.predefined_resources[i].available = - std::min(resources.predefined_resources[i].available + - task_req.predefined_resources[i].demand, - resources.predefined_resources[i].total); - } - - for (const auto &task_req_custom_resource : task_req.custom_resources) { - auto it = resources.custom_resources.find(task_req_custom_resource.id); - if (it != resources.custom_resources.end()) { - it->second.available = std::min( - it->second.available + task_req_custom_resource.demand, it->second.total); - } - } - return true; -} - -bool ClusterResourceScheduler::AddNodeAvailableResources( - const std::string &node_id, - const std::unordered_map &resource_map) { - TaskRequest task_request = ResourceMapToTaskRequest(string_to_int_map_, resource_map); - return AddNodeAvailableResources(string_to_int_map_.Get(node_id), task_request); -} - bool ClusterResourceScheduler::GetNodeResources(int64_t node_id, NodeResources *ret_resources) const { auto it = nodes_.find(node_id); if (it != nodes_.end()) { - *ret_resources = it->second; + *ret_resources = it->second.GetLocalView(); return true; } else { return false; @@ -329,6 +318,36 @@ bool ClusterResourceScheduler::GetNodeResources(int64_t node_id, int64_t ClusterResourceScheduler::NumNodes() { return nodes_.size(); } +void ClusterResourceScheduler::AddLocalResource(const std::string &resource_name, + double resource_total) { + string_to_int_map_.Insert(resource_name); + int64_t resource_id = string_to_int_map_.Get(resource_name); + + if (local_resources_.custom_resources.contains(resource_id)) { + FixedPoint total(resource_total); + auto &instances = local_resources_.custom_resources[resource_id]; + instances.total[0] += total; + instances.available[0] += total; + auto local_node_it = nodes_.find(local_node_id_); + RAY_CHECK(local_node_it != nodes_.end()); + auto &capacity = + local_node_it->second.GetMutableLocalView()->custom_resources[resource_id]; + capacity.available += total; + capacity.total += total; + } else { + ResourceInstanceCapacities capacity; + capacity.total.resize(1); + capacity.total[0] = resource_total; + capacity.available.resize(1); + capacity.available[0] = resource_total; + local_resources_.custom_resources.emplace(resource_id, capacity); + std::string node_id_string = string_to_int_map_.Get(local_node_id_); + RAY_CHECK(string_to_int_map_.Get(node_id_string) == local_node_id_); + UpdateResourceCapacity(node_id_string, resource_name, resource_total); + UpdateLocalAvailableResourcesFromResourceInstances(); + } +} + void ClusterResourceScheduler::UpdateResourceCapacity(const std::string &node_id_string, const std::string &resource_name, double resource_total) { @@ -339,9 +358,7 @@ void ClusterResourceScheduler::UpdateResourceCapacity(const std::string &node_id NodeResources node_resources; node_resources.predefined_resources.resize(PredefinedResources_MAX); node_id = string_to_int_map_.Insert(node_id_string); - RAY_CHECK(nodes_.emplace(node_id, node_resources).second); - it = nodes_.find(node_id); - RAY_CHECK(it != nodes_.end()); + it = nodes_.emplace(node_id, node_resources).first; } int idx = -1; @@ -355,21 +372,23 @@ void ClusterResourceScheduler::UpdateResourceCapacity(const std::string &node_id idx = (int)MEM; }; + auto local_view = it->second.GetMutableLocalView(); FixedPoint resource_total_fp(resource_total); if (idx != -1) { - auto diff_capacity = resource_total_fp - it->second.predefined_resources[idx].total; - it->second.predefined_resources[idx].total += diff_capacity; - it->second.predefined_resources[idx].available += diff_capacity; - if (it->second.predefined_resources[idx].available < 0) { - it->second.predefined_resources[idx].available = 0; + auto diff_capacity = resource_total_fp - local_view->predefined_resources[idx].total; + local_view->predefined_resources[idx].total += diff_capacity; + local_view->predefined_resources[idx].available += diff_capacity; + if (local_view->predefined_resources[idx].available < 0) { + local_view->predefined_resources[idx].available = 0; } - if (it->second.predefined_resources[idx].total < 0) { - it->second.predefined_resources[idx].total = 0; + if (local_view->predefined_resources[idx].total < 0) { + local_view->predefined_resources[idx].total = 0; } } else { - int64_t resource_id = string_to_int_map_.Insert(resource_name); - auto itr = it->second.custom_resources.find(resource_id); - if (itr != it->second.custom_resources.end()) { + string_to_int_map_.Insert(resource_name); + int64_t resource_id = string_to_int_map_.Get(resource_name); + auto itr = local_view->custom_resources.find(resource_id); + if (itr != local_view->custom_resources.end()) { auto diff_capacity = resource_total_fp - itr->second.total; itr->second.total += diff_capacity; itr->second.available += diff_capacity; @@ -382,11 +401,15 @@ void ClusterResourceScheduler::UpdateResourceCapacity(const std::string &node_id } else { ResourceCapacity resource_capacity; resource_capacity.total = resource_capacity.available = resource_total_fp; - it->second.custom_resources.emplace(resource_id, resource_capacity); + local_view->custom_resources.emplace(resource_id, resource_capacity); } } } +void ClusterResourceScheduler::DeleteLocalResource(const std::string &resource_name) { + DeleteResource(string_to_int_map_.Get(local_node_id_), resource_name); +} + void ClusterResourceScheduler::DeleteResource(const std::string &node_id_string, const std::string &resource_name) { int64_t node_id = string_to_int_map_.Get(node_id_string); @@ -405,14 +428,25 @@ void ClusterResourceScheduler::DeleteResource(const std::string &node_id_string, } else if (resource_name == ray::kMemory_ResourceLabel) { idx = (int)MEM; }; + auto local_view = it->second.GetMutableLocalView(); if (idx != -1) { - it->second.predefined_resources[idx].total = 0; + local_view->predefined_resources[idx].total = 0; + + if (node_id == local_node_id_) { + local_resources_.predefined_resources[idx].total.clear(); + local_resources_.predefined_resources[idx].available.clear(); + } } else { int64_t resource_id = string_to_int_map_.Get(resource_name); - auto itr = it->second.custom_resources.find(resource_id); - if (itr != it->second.custom_resources.end()) { + auto itr = local_view->custom_resources.find(resource_id); + if (itr != local_view->custom_resources.end()) { string_to_int_map_.Remove(resource_id); - it->second.custom_resources.erase(itr); + local_view->custom_resources.erase(itr); + } + + if (node_id == local_node_id_) { + local_resources_.custom_resources[resource_id].total.clear(); + local_resources_.custom_resources[resource_id].available.clear(); } } } @@ -423,7 +457,7 @@ std::string ClusterResourceScheduler::DebugString(void) const { buffer << " Local resources: " << local_resources_.DebugString(string_to_int_map_); for (auto &node : nodes_) { buffer << "node id: " << node.first; - buffer << node.second.DebugString(string_to_int_map_); + buffer << node.second.GetLocalView().DebugString(string_to_int_map_); } return buffer.str(); } @@ -641,16 +675,17 @@ void ClusterResourceScheduler::UpdateLocalAvailableResourcesFromResourceInstance auto it_local_node = nodes_.find(local_node_id_); RAY_CHECK(it_local_node != nodes_.end()); + auto local_view = it_local_node->second.GetMutableLocalView(); for (size_t i = 0; i < PredefinedResources_MAX; i++) { - it_local_node->second.predefined_resources[i].available = 0; + local_view->predefined_resources[i].available = 0; for (size_t j = 0; j < local_resources_.predefined_resources[i].available.size(); j++) { - it_local_node->second.predefined_resources[i].available += + local_view->predefined_resources[i].available += local_resources_.predefined_resources[i].available[j]; } } - for (auto &custom_resource : it_local_node->second.custom_resources) { + for (auto &custom_resource : local_view->custom_resources) { auto it = local_resources_.custom_resources.find(custom_resource.first); if (it != local_resources_.custom_resources.end()) { custom_resource.second.available = 0; @@ -746,19 +781,12 @@ std::vector ClusterResourceScheduler::SubtractGPUResourceInstances( return VectorFixedPointToVectorDouble(underflow); } -bool ClusterResourceScheduler::AllocateTaskResources( - int64_t node_id, const TaskRequest &task_req, +bool ClusterResourceScheduler::AllocateLocalTaskResources( + const TaskRequest &task_request, std::shared_ptr task_allocation) { - if (node_id == local_node_id_) { - RAY_CHECK(task_allocation != nullptr); - if (AllocateTaskResourceInstances(task_req, task_allocation)) { - UpdateLocalAvailableResourcesFromResourceInstances(); - return true; - } - } else { - if (SubtractNodeAvailableResources(node_id, task_req)) { - return true; - } + if (AllocateTaskResourceInstances(task_request, task_allocation)) { + UpdateLocalAvailableResourcesFromResourceInstances(); + return true; } return false; } @@ -768,7 +796,7 @@ bool ClusterResourceScheduler::AllocateLocalTaskResources( std::shared_ptr task_allocation) { RAY_CHECK(task_allocation != nullptr); TaskRequest task_request = ResourceMapToTaskRequest(string_to_int_map_, task_resources); - return AllocateTaskResources(local_node_id_, task_request, task_allocation); + return AllocateLocalTaskResources(task_request, task_allocation); } std::string ClusterResourceScheduler::GetResourceNameFromIndex(int64_t res_idx) { @@ -785,13 +813,13 @@ std::string ClusterResourceScheduler::GetResourceNameFromIndex(int64_t res_idx) } } -void ClusterResourceScheduler::AllocateRemoteTaskResources( +bool ClusterResourceScheduler::AllocateRemoteTaskResources( const std::string &node_string, const std::unordered_map &task_resources) { TaskRequest task_request = ResourceMapToTaskRequest(string_to_int_map_, task_resources); auto node_id = string_to_int_map_.Insert(node_string); RAY_CHECK(node_id != local_node_id_); - AllocateTaskResources(node_id, task_request, nullptr); + return SubtractRemoteNodeAvailableResources(node_id, task_request); } void ClusterResourceScheduler::FreeLocalTaskResources( @@ -803,27 +831,26 @@ void ClusterResourceScheduler::FreeLocalTaskResources( UpdateLocalAvailableResourcesFromResourceInstances(); } -void ClusterResourceScheduler::Heartbeat( - bool light_heartbeat_enabled, std::shared_ptr heartbeat_data) { +void ClusterResourceScheduler::FillResourceUsage( + bool light_report_resource_usage_enabled, + std::shared_ptr resources_data) { NodeResources resources; RAY_CHECK(GetNodeResources(local_node_id_, &resources)) << "Error: Populating heartbeat failed. Please file a bug report: " "https://github.com/ray-project/ray/issues/new."; - if (light_heartbeat_enabled && last_report_resources_ && - resources == *last_report_resources_.get()) { - return; - } else { + if (!light_report_resource_usage_enabled || !last_report_resources_ || + resources != *last_report_resources_.get()) { for (int i = 0; i < PredefinedResources_MAX; i++) { const auto &label = ResourceEnumToString((PredefinedResources)i); const auto &capacity = resources.predefined_resources[i]; if (capacity.available != 0) { - (*heartbeat_data->mutable_resources_available())[label] = + (*resources_data->mutable_resources_available())[label] = capacity.available.Double(); } if (capacity.total != 0) { - (*heartbeat_data->mutable_resources_total())[label] = capacity.total.Double(); + (*resources_data->mutable_resources_total())[label] = capacity.total.Double(); } } for (auto it = resources.custom_resources.begin(); @@ -832,18 +859,30 @@ void ClusterResourceScheduler::Heartbeat( const auto &capacity = it->second; const auto &label = string_to_int_map_.Get(custom_id); if (capacity.available != 0) { - (*heartbeat_data->mutable_resources_available())[label] = + (*resources_data->mutable_resources_available())[label] = capacity.available.Double(); } if (capacity.total != 0) { - (*heartbeat_data->mutable_resources_total())[label] = capacity.total.Double(); + (*resources_data->mutable_resources_total())[label] = capacity.total.Double(); } } - heartbeat_data->set_resources_available_changed(true); - if (light_heartbeat_enabled) { + resources_data->set_resources_available_changed(true); + if (light_report_resource_usage_enabled) { last_report_resources_.reset(new NodeResources(resources)); } } + + if (light_report_resource_usage_enabled) { + // Reset all local views for remote nodes. This is needed in case tasks that + // we spilled back to a remote node were not actually scheduled on the + // node. Then, the remote node's resource availability may not change and + // so it may not send us another update. + for (auto &node : nodes_) { + if (node.first != local_node_id_) { + node.second.ResetLocalView(); + } + } + } } } // namespace ray diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler.h b/src/ray/raylet/scheduling/cluster_resource_scheduler.h index c476e174a..dfaa61fbd 100644 --- a/src/ray/raylet/scheduling/cluster_resource_scheduler.h +++ b/src/ray/raylet/scheduling/cluster_resource_scheduler.h @@ -39,46 +39,6 @@ static std::unordered_set UnitInstanceResources{CPU, GPU, TPU}; /// tasks to nodes based on the task's constraints and the available /// resources at those nodes. class ClusterResourceScheduler { - /// List of nodes in the clusters and their resources organized as a map. - /// The key of the map is the node ID. - absl::flat_hash_map nodes_; - /// Identifier of local node. - int64_t local_node_id_; - /// Resources of local node. - NodeResourceInstances local_resources_; - /// Keep the mapping between node and resource IDs in string representation - /// to integer representation. Used for improving map performance. - StringIdMap string_to_int_map_; - /// Cached resources, used to compare with newest one in light heartbeat mode. - std::unique_ptr last_report_resources_; - - /// Set predefined resources. - /// - /// \param[in] new_resources: New predefined resources. - /// \param[out] old_resources: Predefined resources to be updated. - void SetPredefinedResources(const NodeResources &new_resources, - NodeResources *old_resources); - /// Set custom resources. - /// - /// \param[in] new_resources: New custom resources. - /// \param[out] old_resources: Custom resources to be updated. - void SetCustomResources( - const absl::flat_hash_map &new_custom_resources, - absl::flat_hash_map *old_custom_resources); - - /// Subtract the resources required by a given task request (task_req) from - /// a given node (node_id). - /// - /// \param node_id Node whose resources we allocate. Can be the local or a remote node. - /// \param task_req Task for which we allocate resources. - /// \param task_allocation Resources allocated to the task at instance granularity. - /// This is a return parameter. - /// - /// \return True if the node has enough resources to satisfy the task request. - /// False otherwise. - bool AllocateTaskResources(int64_t node_id, const TaskRequest &task_req, - std::shared_ptr task_allocation); - public: ClusterResourceScheduler(void){}; @@ -113,6 +73,21 @@ class ClusterResourceScheduler { bool RemoveNode(int64_t node_id); bool RemoveNode(const std::string &node_id_string); + /// Check whether a task request is feasible on a given node. A node is + /// feasible if it has the total resources needed to eventually execute the + /// task, even if those resources are currently allocated. + /// + /// \param shape The resource demand's shape. + bool IsLocallyFeasible(const std::unordered_map shape); + + /// Check whether a task request is feasible on a given node. A node is + /// feasible if it has the total resources needed to eventually execute the + /// task, even if those resources are currently allocated. + /// + /// \param task_req Task request to be scheduled. + /// \param resources Node's resources. + bool IsFeasible(const TaskRequest &task_req, const NodeResources &resources) const; + /// Check whether a task request can be scheduled given a node. /// /// \param task_req: Task request to be scheduled. @@ -128,23 +103,29 @@ class ClusterResourceScheduler { /// >= 0, the number soft constraint violations. If 0, no /// constraint is violated. int64_t IsSchedulable(const TaskRequest &task_req, int64_t node_id, - const NodeResources &resources); + const NodeResources &resources) const; /// Find a node in the cluster on which we can schedule a given task request. /// - /// First, this function checks whether the local node can schedule - /// the request without violating any constraints. If yes, it returns the - /// ID of the local node. + /// Ignoring soft constraints, this policy prioritizes nodes in the + /// following order: /// - /// If not, this function checks whether there is another node in the cluster - /// that satisfies all request's constraints (both soft and hard). + /// 1. Local node if resources available. + /// 2. Any remote node if resources available. + /// 3. If the local node is not feasible, any remote node if feasible. /// - /// If no such node exists, the function checks whether there are nodes - /// that satisfy all the request's hard constraints, but might violate some - /// soft constraints. Among these nodes, it returns a node which violates - /// the least number of soft constraints. + /// If soft constraints are specified, then this policy will prioritize: + /// 1. Local node if resources available and does not violate soft + /// constraints. + /// 2. Any remote node if resources available and does not violate soft + /// constraints. + /// 3. Out of all the nodes, including the local node, pick the one that + /// has resources available and violates the fewest soft constraints. + /// 4. If the local node is not feasible, any remote node if feasible. /// - /// Finally, if no such node exists, return -1. + /// If no node can meet any of these, returns -1, in which case the caller + /// should queue the task and try again once resource availability has been + /// updated. /// /// \param task_request: Task to be scheduled. /// \param actor_creation: True if this is an actor creation task. @@ -168,32 +149,6 @@ class ClusterResourceScheduler { const std::unordered_map &task_request, bool actor_creation, int64_t *violations); - /// Decrease the available resources of a node when a task request is - /// scheduled on the given node. - /// - /// \param node_id: ID of node on which request is being scheduled. - /// \param task_req: task request being scheduled. - /// - /// \return true, if task_req can be indeed scheduled on the node, - /// and false otherwise. - bool SubtractNodeAvailableResources(int64_t node_id, const TaskRequest &task_request); - bool SubtractNodeAvailableResources( - const std::string &node_id, - const std::unordered_map &task_request); - - /// Increase available resources of a node when a worker has finished - /// a task. - /// - /// \param node_id: ID of node on which request is being scheduled. - /// \param task_request: resource requests of the task finishing execution. - /// - /// \return true, if task_req can be indeed scheduled on the node, - /// and false otherwise. - bool AddNodeAvailableResources(int64_t node_id, const TaskRequest &task_request); - bool AddNodeAvailableResources( - const std::string &node_id, - const std::unordered_map &task_request); - /// Return resources associated to the given node_id in ret_resources. /// If node_id not found, return false; otherwise return true. bool GetNodeResources(int64_t node_id, NodeResources *ret_resources) const; @@ -201,6 +156,12 @@ class ClusterResourceScheduler { /// Get number of nodes in the cluster. int64_t NumNodes(); + /// Add a local resource that is available. + /// + /// \param resource_name: Resource which we want to update. + /// \param resource_total: New capacity of the resource. + void AddLocalResource(const std::string &resource_name, double resource_total); + /// Update total capacity of a given resource of a given node. /// /// \param node_name: Node whose resource we want to update. @@ -209,6 +170,11 @@ class ClusterResourceScheduler { void UpdateResourceCapacity(const std::string &node_name, const std::string &resource_name, double resource_total); + /// Delete a given resource from the local node. + /// + /// \param resource_name: Resource we want to delete + void DeleteLocalResource(const std::string &resource_name); + /// Delete a given resource from a given node. /// /// \param node_name: Node whose resource we want to delete. @@ -355,12 +321,17 @@ class ClusterResourceScheduler { const std::unordered_map &task_resources, std::shared_ptr task_allocation); + bool AllocateLocalTaskResources(const TaskRequest &task_request, + std::shared_ptr task_allocation); + /// Subtract the resources required by a given task request (task_req) from a given /// remote node. /// /// \param node_id Remote node whose resources we allocate. /// \param task_req Task for which we allocate resources. - void AllocateRemoteTaskResources( + /// \return True if remote node has enough resources to satisfy the task request. + /// False otherwise. + bool AllocateRemoteTaskResources( const std::string &node_id, const std::unordered_map &task_resources); @@ -381,13 +352,65 @@ class ClusterResourceScheduler { /// sending raylet <-> gcs heartbeats. In particular, this should fill in /// resources_available and resources_total. /// - /// \param light_heartbeat_enabled Only send changed fields if true. + /// \param light_report_resource_usage_enabled Only send changed fields if true. /// \param Output parameter. `resources_available` and `resources_total` are the only /// fields used. - void Heartbeat(bool light_heartbeat_enabled, std::shared_ptr data); + void FillResourceUsage(bool light_report_resource_usage_enabled, + std::shared_ptr resources_data); /// Return human-readable string for this scheduler state. std::string DebugString() const; + + private: + struct Node { + Node(const NodeResources &resources) + : last_reported_(resources), local_view_(resources) {} + + void ResetLocalView() { local_view_ = last_reported_; } + + NodeResources *GetMutableLocalView() { return &local_view_; } + + const NodeResources &GetLocalView() const { return local_view_; } + + private: + /// The resource information according to the last heartbeat reported by + /// this node. + /// NOTE(swang): For the local node, this field should be ignored because + /// we do not receive heartbeats from ourselves and the local view is + /// therefore always the most up-to-date. + NodeResources last_reported_; + /// Our local view of the remote node's resources. This may be dirty + /// because it includes any resource requests that we allocated to this + /// node through spillback since our last heartbeat tick. This view will + /// get overwritten by the last reported view on each heartbeat tick, to + /// make sure that our local view does not skew too much from the actual + /// resources when light heartbeats are enabled. + NodeResources local_view_; + }; + + /// Decrease the available resources of a node when a task request is + /// scheduled on the given node. + /// + /// \param node_id: ID of node on which request is being scheduled. + /// \param task_req: task request being scheduled. + /// + /// \return true, if task_req can be indeed scheduled on the node, + /// and false otherwise. + bool SubtractRemoteNodeAvailableResources(int64_t node_id, + const TaskRequest &task_request); + + /// List of nodes in the clusters and their resources organized as a map. + /// The key of the map is the node ID. + absl::flat_hash_map nodes_; + /// Identifier of local node. + int64_t local_node_id_; + /// Resources of local node. + NodeResourceInstances local_resources_; + /// Keep the mapping between node and resource IDs in string representation + /// to integer representation. Used for improving map performance. + StringIdMap string_to_int_map_; + /// Cached resources, used to compare with newest one in light heartbeat mode. + std::unique_ptr last_report_resources_; }; } // end namespace ray diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc b/src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc index 08a8c7d30..a43ed2e4c 100644 --- a/src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc +++ b/src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc @@ -29,6 +29,21 @@ using namespace std; +#define ASSERT_RESOURCES_EQ(data, expected_available, expected_total) \ + { \ + auto available = data->resources_available(); \ + ASSERT_EQ(available[kCPU_ResourceLabel], expected_available); \ + auto total = data->resources_total(); \ + ASSERT_EQ(total[kCPU_ResourceLabel], expected_total); \ + } + +#define ASSERT_RESOURCES_EMPTY(data) \ + { \ + ASSERT_FALSE(data->resources_available_changed()); \ + ASSERT_TRUE(data->resources_available().empty()); \ + ASSERT_TRUE(data->resources_total().empty()); \ + } + namespace ray { // Used to path empty vector argiuments. vector EmptyIntVector; @@ -334,11 +349,13 @@ TEST_F(ClusterResourceSchedulerTest, SchedulingUpdateAvailableResourcesTest) { int64_t node_id = cluster_resources.GetBestSchedulableNode(task_req, false, &violations); ASSERT_TRUE(node_id != -1); + ASSERT_EQ(node_id, 1); ASSERT_TRUE(violations > 0); NodeResources nr1, nr2; ASSERT_TRUE(cluster_resources.GetNodeResources(node_id, &nr1)); - cluster_resources.SubtractNodeAvailableResources(node_id, task_req); + auto task_allocation = std::make_shared(); + ASSERT_TRUE(cluster_resources.AllocateLocalTaskResources(task_req, task_allocation)); ASSERT_TRUE(cluster_resources.GetNodeResources(node_id, &nr2)); for (size_t i = 0; i < PRED_CUSTOM_LEN; i++) { @@ -981,6 +998,34 @@ TEST_F(ClusterResourceSchedulerTest, TaskResourceInstanceWithHardRequestTest) { ASSERT_TRUE(EqualVectors(cpu_instances, expect_cpu_instance)); } +TEST_F(ClusterResourceSchedulerTest, TestAlwaysSpillInfeasibleTask) { + std::unordered_map resource_spec({{"CPU", 1}}); + ClusterResourceScheduler cluster_resources("local", {}); + for (int i = 0; i < 100; i++) { + cluster_resources.AddOrUpdateNode(std::to_string(i), {}, {}); + } + + // No feasible nodes. + int64_t total_violations; + ASSERT_EQ( + cluster_resources.GetBestSchedulableNode(resource_spec, false, &total_violations), + ""); + + // Feasible remote node, but doesn't currently have resources available. We + // should spill there. + cluster_resources.AddOrUpdateNode("remote_feasible", resource_spec, {{"CPU", 0.}}); + ASSERT_EQ( + cluster_resources.GetBestSchedulableNode(resource_spec, false, &total_violations), + "remote_feasible"); + + // Feasible remote node, and it currently has resources available. We should + // prefer to spill there. + cluster_resources.AddOrUpdateNode("remote_available", resource_spec, resource_spec); + ASSERT_EQ( + cluster_resources.GetBestSchedulableNode(resource_spec, false, &total_violations), + "remote_available"); +} + TEST_F(ClusterResourceSchedulerTest, HeartbeatTest) { vector cust_ids{1, 2, 3, 4, 5}; @@ -997,8 +1042,8 @@ TEST_F(ClusterResourceSchedulerTest, HeartbeatTest) { cluster_resources.AddOrUpdateNode(12345, other_node_resources); { // Cluster is idle. - auto data = std::make_shared(); - cluster_resources.Heartbeat(false, data); + auto data = std::make_shared(); + cluster_resources.FillResourceUsage(false, data); auto available = data->resources_available(); auto total = data->resources_total(); @@ -1035,8 +1080,8 @@ TEST_F(ClusterResourceSchedulerTest, HeartbeatTest) { {"1", 0.1}, }); cluster_resources.AllocateLocalTaskResources(allocation_map, allocations); - auto data = std::make_shared(); - cluster_resources.Heartbeat(false, data); + auto data = std::make_shared(); + cluster_resources.FillResourceUsage(false, data); auto available = data->resources_available(); auto total = data->resources_total(); @@ -1057,6 +1102,106 @@ TEST_F(ClusterResourceSchedulerTest, HeartbeatTest) { } } +TEST_F(ClusterResourceSchedulerTest, TestLightResourceUsageReport) { + std::unordered_map initial_resources({{"CPU", 1}}); + ClusterResourceScheduler cluster_resources("local", initial_resources); + + // Fill resource usage usage on initialization. + auto data = std::make_shared(); + cluster_resources.FillResourceUsage(true, data); + ASSERT_RESOURCES_EQ(data, 1, 1); + + // Don't report resource usage if resource availability hasn't changed. + for (int i = 0; i < 3; i++) { + data->Clear(); + cluster_resources.FillResourceUsage(true, data); + ASSERT_RESOURCES_EMPTY(data); + } + + // Report resource usage if resource availability has changed. + cluster_resources.AddOrUpdateNode("local", {{"CPU", 1.}}, {{"CPU", 0.}}); + data->Clear(); + cluster_resources.FillResourceUsage(true, data); + ASSERT_RESOURCES_EQ(data, 0, 1); + + // Don't report resource usage if resource availability hasn't changed. + for (int i = 0; i < 3; i++) { + data->Clear(); + cluster_resources.FillResourceUsage(true, data); + ASSERT_RESOURCES_EMPTY(data); + } +} + +TEST_F(ClusterResourceSchedulerTest, TestDirtyLocalView) { + std::unordered_map initial_resources({{"CPU", 1}}); + ClusterResourceScheduler cluster_resources("local", initial_resources); + cluster_resources.AddOrUpdateNode("remote", {{"CPU", 2.}}, {{"CPU", 2.}}); + const std::unordered_map task_spec = {{"CPU", 1.}}; + + // Allocate local resources to force tasks onto the remote node when + // resources are available. + std::shared_ptr task_allocation = + std::make_shared(); + ASSERT_TRUE(cluster_resources.AllocateLocalTaskResources(task_spec, task_allocation)); + task_allocation = std::make_shared(); + ASSERT_FALSE(cluster_resources.AllocateLocalTaskResources(task_spec, task_allocation)); + // View of local resources is not affected by resource usage report. + auto data = std::make_shared(); + cluster_resources.FillResourceUsage(true, data); + ASSERT_FALSE(cluster_resources.AllocateLocalTaskResources(task_spec, task_allocation)); + + for (int num_slots_available = 0; num_slots_available <= 2; num_slots_available++) { + // Remote node reports updated resource availability. + cluster_resources.AddOrUpdateNode("remote", {{"CPU", 2.}}, + {{"CPU", num_slots_available}}); + auto data = std::make_shared(); + int64_t t; + for (int i = 0; i < 3; i++) { + // Resource usage report tick should reset the remote node's resources. + cluster_resources.FillResourceUsage(true, data); + for (int j = 0; j < num_slots_available; j++) { + ASSERT_EQ(cluster_resources.GetBestSchedulableNode(task_spec, false, &t), + "remote"); + // Allocate remote resources. + ASSERT_TRUE(cluster_resources.AllocateRemoteTaskResources("remote", task_spec)); + } + // Our local view says there are not enough resources on the remote node to + // schedule another task. + ASSERT_EQ(cluster_resources.GetBestSchedulableNode(task_spec, false, &t), ""); + ASSERT_FALSE( + cluster_resources.AllocateLocalTaskResources(task_spec, task_allocation)); + ASSERT_FALSE(cluster_resources.AllocateRemoteTaskResources("remote", task_spec)); + } + } +} + +TEST_F(ClusterResourceSchedulerTest, DynamicResourceTest) { + ClusterResourceScheduler cluster_resources("local", {{"CPU", 2}}); + + std::unordered_map task_request = {{"CPU", 1}, {"custom123", 2}}; + int64_t t; + + std::string result = cluster_resources.GetBestSchedulableNode(task_request, false, &t); + ASSERT_TRUE(result.empty()); + + cluster_resources.AddLocalResource("custom123", 5); + + result = cluster_resources.GetBestSchedulableNode(task_request, false, &t); + ASSERT_FALSE(result.empty()); + + task_request["custom123"] = 6; + result = cluster_resources.GetBestSchedulableNode(task_request, false, &t); + ASSERT_TRUE(result.empty()); + + cluster_resources.AddLocalResource("custom123", 5); + result = cluster_resources.GetBestSchedulableNode(task_request, false, &t); + ASSERT_FALSE(result.empty()); + + cluster_resources.DeleteLocalResource("custom123"); + result = cluster_resources.GetBestSchedulableNode(task_request, false, &t); + ASSERT_TRUE(result.empty()); +} + } // namespace ray int main(int argc, char **argv) { diff --git a/src/ray/raylet/scheduling/cluster_task_manager.cc b/src/ray/raylet/scheduling/cluster_task_manager.cc index c77a89726..4b6e5e0d7 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.cc +++ b/src/ray/raylet/scheduling/cluster_task_manager.cc @@ -17,7 +17,10 @@ ClusterTaskManager::ClusterTaskManager( cluster_resource_scheduler_(cluster_resource_scheduler), fulfills_dependencies_func_(fulfills_dependencies_func), is_owner_alive_(is_owner_alive), - get_node_info_(get_node_info) {} + get_node_info_(get_node_info), + max_resource_shapes_per_load_report_( + RayConfig::instance().max_resource_shapes_per_load_report()), + report_worker_backlog_(RayConfig::instance().report_worker_backlog()) {} bool ClusterTaskManager::SchedulePendingTasks() { bool did_schedule = false; @@ -45,6 +48,8 @@ bool ClusterTaskManager::SchedulePendingTasks() { if (node_id_string.empty()) { // There is no node that has available resources to run the request. // Move on to the next shape. + RAY_LOG(DEBUG) << "No feasible node found for task " + << task.GetTaskSpecification().TaskId(); break; } else { if (node_id_string == self_node_id_.Binary()) { @@ -81,6 +86,8 @@ bool ClusterTaskManager::WaitForTaskArgsRequests(Work work) { << task.GetTaskSpecification().TaskId(); tasks_to_dispatch_[scheduling_key].push_back(work); } else { + RAY_LOG(DEBUG) << "Waiting for args for task: " + << task.GetTaskSpecification().TaskId(); can_dispatch = false; TaskID task_id = task.GetTaskSpecification().TaskId(); waiting_tasks_[task_id] = work; @@ -130,7 +137,7 @@ void ClusterTaskManager::DispatchScheduledTasksToWorkers( if (worker_leased) { auto reply = std::get<1>(*work_it); auto callback = std::get<2>(*work_it); - Dispatch(worker, leased_workers, spec, reply, callback); + Dispatch(worker, leased_workers, task, reply, callback); } else { worker_pool.PushWorker(worker); } @@ -182,9 +189,6 @@ bool ClusterTaskManager::AttemptDispatchWork(const Work &work, worker->SetAllocatedInstances(allocated_instances); } worker->AssignTaskId(spec.TaskId()); - if (!RayConfig::instance().enable_multi_tenancy()) { - worker->AssignJobId(spec.JobId()); - } worker->SetAssignedTask(task); *worker_leased = true; dispatched = true; @@ -198,6 +202,7 @@ void ClusterTaskManager::QueueTask(const Task &task, rpc::RequestWorkerLeaseRepl Work work = std::make_tuple(task, reply, callback); const auto &scheduling_class = task.GetTaskSpecification().GetSchedulingClass(); tasks_to_schedule_[scheduling_class].push_back(work); + AddToBacklogTracker(task); } void ClusterTaskManager::TasksUnblocked(const std::vector ready_ids) { @@ -235,7 +240,10 @@ bool ClusterTaskManager::CancelTask(const TaskID &task_id) { shapes_it++) { auto &work_queue = shapes_it->second; for (auto work_it = work_queue.begin(); work_it != work_queue.end(); work_it++) { - if (std::get<0>(*work_it).GetTaskSpecification().TaskId() == task_id) { + const auto &task = std::get<0>(*work_it); + if (task.GetTaskSpecification().TaskId() == task_id) { + RemoveFromBacklogTracker(task); + RAY_LOG(DEBUG) << "Canceling task " << task_id; ReplyCancelled(*work_it); work_queue.erase(work_it); if (work_queue.empty()) { @@ -249,7 +257,9 @@ bool ClusterTaskManager::CancelTask(const TaskID &task_id) { shapes_it++) { auto &work_queue = shapes_it->second; for (auto work_it = work_queue.begin(); work_it != work_queue.end(); work_it++) { - if (std::get<0>(*work_it).GetTaskSpecification().TaskId() == task_id) { + const auto &task = std::get<0>(*work_it); + if (task.GetTaskSpecification().TaskId() == task_id) { + RemoveFromBacklogTracker(task); ReplyCancelled(*work_it); work_queue.erase(work_it); if (work_queue.empty()) { @@ -262,6 +272,8 @@ bool ClusterTaskManager::CancelTask(const TaskID &task_id) { auto iter = waiting_tasks_.find(task_id); if (iter != waiting_tasks_.end()) { + const auto &task = std::get<0>(iter->second); + RemoveFromBacklogTracker(task); ReplyCancelled(iter->second); waiting_tasks_.erase(iter); return true; @@ -270,8 +282,12 @@ bool ClusterTaskManager::CancelTask(const TaskID &task_id) { return false; } -void ClusterTaskManager::Heartbeat(bool light_heartbeat_enabled, - std::shared_ptr data) const { +void ClusterTaskManager::FillResourceUsage( + bool light_report_resource_usage_enabled, + std::shared_ptr data) const { + if (max_resource_shapes_per_load_report_ == 0) { + return; + } // TODO (WangTao): Find a way to check if load changed and combine it with light // heartbeat. Now we just report it every time. data->set_resource_load_changed(true); @@ -279,9 +295,59 @@ void ClusterTaskManager::Heartbeat(bool light_heartbeat_enabled, auto resource_load_by_shape = data->mutable_resource_load_by_shape()->mutable_resource_demands(); - // TODO (Alex): Implement the 1-CPU task optimization. + int num_reported = 0; + + // 1-CPU optimization + static const ResourceSet one_cpu_resource_set( + std::unordered_map({{kCPU_ResourceLabel, 1}})); + static const SchedulingClass one_cpu_scheduling_cls( + TaskSpecification::GetSchedulingClass(one_cpu_resource_set)); + { + num_reported++; + int count = 0; + auto it = tasks_to_schedule_.find(one_cpu_scheduling_cls); + if (it != tasks_to_schedule_.end()) { + count += it->second.size(); + } + it = tasks_to_dispatch_.find(one_cpu_scheduling_cls); + if (it != tasks_to_dispatch_.end()) { + count += it->second.size(); + } + + if (count > 0) { + auto by_shape_entry = resource_load_by_shape->Add(); + + for (const auto &resource : one_cpu_resource_set.GetResourceMap()) { + // Add to `resource_loads`. + const auto &label = resource.first; + const auto &quantity = resource.second; + (*resource_loads)[label] += quantity * count; + + // Add to `resource_load_by_shape`. + (*by_shape_entry->mutable_shape())[label] = quantity; + } + + int num_ready = by_shape_entry->num_ready_requests_queued(); + by_shape_entry->set_num_ready_requests_queued(num_ready + count); + + auto backlog_it = backlog_tracker_.find(one_cpu_scheduling_cls); + if (backlog_it != backlog_tracker_.end()) { + by_shape_entry->set_backlog_size(backlog_it->second); + } + } + } + for (const auto &pair : tasks_to_schedule_) { const auto &scheduling_class = pair.first; + if (scheduling_class == one_cpu_scheduling_cls) { + continue; + } + if (num_reported++ >= max_resource_shapes_per_load_report_ && + max_resource_shapes_per_load_report_ >= 0) { + // TODO (Alex): It's possible that we skip a different scheduling key which contains + // the same resources. + break; + } const auto &resources = TaskSpecification::GetSchedulingClassDescriptor(scheduling_class) .GetResourceMap(); @@ -298,14 +364,35 @@ void ClusterTaskManager::Heartbeat(bool light_heartbeat_enabled, // Add to `resource_load_by_shape`. (*by_shape_entry->mutable_shape())[label] = quantity; - // TODO (Alex): Technically being on `tasks_to_schedule` could also mean - // that the entire cluster is utilized. - by_shape_entry->set_num_infeasible_requests_queued(count); + } + + // If a task is not feasible on the local node it will not be feasible on any other + // node in the cluster. See the scheduling policy defined by + // ClusterResourceScheduler::GetBestSchedulableNode for more details. + if (cluster_resource_scheduler_->IsLocallyFeasible(resources)) { + int num_ready = by_shape_entry->num_ready_requests_queued(); + by_shape_entry->set_num_ready_requests_queued(num_ready + count); + } else { + int num_infeasible = by_shape_entry->num_infeasible_requests_queued(); + by_shape_entry->set_num_infeasible_requests_queued(num_infeasible + count); + } + auto backlog_it = backlog_tracker_.find(scheduling_class); + if (backlog_it != backlog_tracker_.end()) { + by_shape_entry->set_backlog_size(backlog_it->second); } } for (const auto &pair : tasks_to_dispatch_) { const auto &scheduling_class = pair.first; + if (scheduling_class == one_cpu_scheduling_cls) { + continue; + } + if (num_reported++ >= max_resource_shapes_per_load_report_ && + max_resource_shapes_per_load_report_ >= 0) { + // TODO (Alex): It's possible that we skip a different scheduling key which contains + // the same resources. + break; + } const auto &resources = TaskSpecification::GetSchedulingClassDescriptor(scheduling_class) .GetResourceMap(); @@ -322,9 +409,12 @@ void ClusterTaskManager::Heartbeat(bool light_heartbeat_enabled, // Add to `resource_load_by_shape`. (*by_shape_entry->mutable_shape())[label] = quantity; - // TODO (Alex): Technically being on `tasks_to_schedule` could also mean - // that the entire cluster is utilized. - by_shape_entry->set_num_ready_requests_queued(count); + } + int num_ready = by_shape_entry->num_ready_requests_queued(); + by_shape_entry->set_num_ready_requests_queued(num_ready + count); + auto backlog_it = backlog_tracker_.find(scheduling_class); + if (backlog_it != backlog_tracker_.end()) { + by_shape_entry->set_backlog_size(backlog_it->second); } } } @@ -344,8 +434,9 @@ std::string ClusterTaskManager::DebugString() const { void ClusterTaskManager::Dispatch( std::shared_ptr worker, std::unordered_map> &leased_workers, - const TaskSpecification &task_spec, rpc::RequestWorkerLeaseReply *reply, + const Task &task, rpc::RequestWorkerLeaseReply *reply, std::function send_reply_callback) { + const auto &task_spec = task.GetTaskSpecification(); RAY_LOG(DEBUG) << "Dispatching task " << task_spec.TaskId(); // Pass the contact info of the worker to use. reply->mutable_worker_address()->set_ip_address(worker->IpAddress()); @@ -409,8 +500,12 @@ void ClusterTaskManager::Dispatch( void ClusterTaskManager::Spillback(const NodeID &spillback_to, const Work &work) { const auto &task_spec = std::get<0>(work).GetTaskSpecification(); RAY_LOG(DEBUG) << "Spilling task " << task_spec.TaskId() << " to node " << spillback_to; - cluster_resource_scheduler_->AllocateRemoteTaskResources( - spillback_to.Binary(), task_spec.GetRequiredResources().GetResourceMap()); + + if (!cluster_resource_scheduler_->AllocateRemoteTaskResources( + spillback_to.Binary(), task_spec.GetRequiredResources().GetResourceMap())) { + RAY_LOG(INFO) << "Tried to allocate resources for request " << task_spec.TaskId() + << " on a remote node that are no longer available"; + } auto node_info_opt = get_node_info_(spillback_to); RAY_CHECK(node_info_opt) @@ -426,5 +521,22 @@ void ClusterTaskManager::Spillback(const NodeID &spillback_to, const Work &work) send_reply_callback(); } +void ClusterTaskManager::AddToBacklogTracker(const Task &task) { + if (report_worker_backlog_) { + auto cls = task.GetTaskSpecification().GetSchedulingClass(); + backlog_tracker_[cls] += task.BacklogSize(); + } +} + +void ClusterTaskManager::RemoveFromBacklogTracker(const Task &task) { + if (report_worker_backlog_) { + SchedulingClass cls = task.GetTaskSpecification().GetSchedulingClass(); + backlog_tracker_[cls] -= task.BacklogSize(); + if (backlog_tracker_[cls] == 0) { + backlog_tracker_.erase(backlog_tracker_.find(cls)); + } + } +} + } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/scheduling/cluster_task_manager.h b/src/ray/raylet/scheduling/cluster_task_manager.h index c5dba73be..6ec2db994 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.h +++ b/src/ray/raylet/scheduling/cluster_task_manager.h @@ -107,11 +107,11 @@ class ClusterTaskManager { /// sending raylet <-> gcs heartbeats. In particular, this should fill in /// resource_load and resource_load_by_shape. /// - /// \param light_heartbeat_enabled Only send changed fields if true. + /// \param light_report_resource_usage_enabled Only send changed fields if true. /// \param Output parameter. `resource_load` and `resource_load_by_shape` are the only /// fields used. - void Heartbeat(bool light_heartbeat_enabled, - std::shared_ptr data) const; + void FillResourceUsage(bool light_report_resource_usage_enabled, + std::shared_ptr data) const; std::string DebugString() const; @@ -128,14 +128,24 @@ class ClusterTaskManager { std::function is_owner_alive_; NodeInfoGetter get_node_info_; + const int max_resource_shapes_per_load_report_; + const bool report_worker_backlog_; + /// Queue of lease requests that are waiting for resources to become available. + /// Tasks move from scheduled -> dispatch | waiting. std::unordered_map> tasks_to_schedule_; /// Queue of lease requests that should be scheduled onto workers. + /// Tasks move from scheduled | waiting -> dispatch. std::unordered_map> tasks_to_dispatch_; + /// Tasks waiting for arguments to be transferred locally. + /// Tasks move from waiting -> dispatch. absl::flat_hash_map waiting_tasks_; + /// Track the cumulative backlog of all workers requesting a lease to this raylet. + std::unordered_map backlog_tracker_; + /// Determine whether a task should be immediately dispatched, /// or placed on a wait queue. /// @@ -145,10 +155,13 @@ class ClusterTaskManager { void Dispatch( std::shared_ptr worker, std::unordered_map> &leased_workers_, - const TaskSpecification &task_spec, rpc::RequestWorkerLeaseReply *reply, + const Task &task, rpc::RequestWorkerLeaseReply *reply, std::function send_reply_callback); void Spillback(const NodeID &spillback_to, const Work &work); + + void AddToBacklogTracker(const Task &task); + void RemoveFromBacklogTracker(const Task &task); }; } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/scheduling/cluster_task_manager_test.cc b/src/ray/raylet/scheduling/cluster_task_manager_test.cc index f20515353..ddda8fed5 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager_test.cc +++ b/src/ray/raylet/scheduling/cluster_task_manager_test.cc @@ -80,7 +80,7 @@ Task CreateTask(const std::unordered_map &required_resource FunctionDescriptorBuilder::BuildPython("", "", "", ""), job_id, TaskID::Nil(), 0, TaskID::Nil(), address, 0, required_resources, {}, - std::make_pair(PlacementGroupID::Nil(), -1), true); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); for (int i = 0; i < num_args; i++) { ObjectID put_id = ObjectID::FromIndex(TaskID::Nil(), /*index=*/i + 1); @@ -437,8 +437,8 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { } { - auto data = std::make_shared(); - task_manager_.Heartbeat(false, data); + auto data = std::make_shared(); + task_manager_.FillResourceUsage(false, data); auto load_by_shape = data->mutable_resource_load_by_shape()->mutable_resource_demands(); @@ -480,6 +480,63 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { } } +TEST_F(ClusterTaskManagerTest, BacklogReportTest) { + /* + Test basic scheduler functionality: + 1. Queue and attempt to schedule/dispatch atest with no workers available + 2. A worker becomes available, dispatch again. + */ + rpc::RequestWorkerLeaseReply reply; + bool callback_occurred = false; + bool *callback_occurred_ptr = &callback_occurred; + auto callback = [callback_occurred_ptr]() { *callback_occurred_ptr = true; }; + + std::shared_ptr worker = + std::make_shared(WorkerID::FromRandom(), 1234); + pool_.PushWorker(std::dynamic_pointer_cast(worker)); + + std::vector to_cancel; + + for (int i = 0; i < 10; i++) { + Task task = CreateTask({{ray::kCPU_ResourceLabel, 100}}); + task.SetBacklogSize(i); + task_manager_.QueueTask(task, &reply, callback); + to_cancel.push_back(task.GetTaskSpecification().TaskId()); + } + task_manager_.SchedulePendingTasks(); + task_manager_.DispatchScheduledTasksToWorkers(pool_, leased_workers_); + + ASSERT_FALSE(callback_occurred); + ASSERT_EQ(leased_workers_.size(), 0); + ASSERT_EQ(pool_.workers.size(), 1); + ASSERT_EQ(fulfills_dependencies_calls_, 0); + ASSERT_EQ(node_info_calls_, 0); + + auto data = std::make_shared(); + task_manager_.FillResourceUsage(false, data); + + auto resource_load_by_shape = data->resource_load_by_shape(); + auto shape1 = resource_load_by_shape.resource_demands()[0]; + + ASSERT_EQ(shape1.backlog_size(), 45); + ASSERT_EQ(shape1.num_infeasible_requests_queued(), 10); + ASSERT_EQ(shape1.num_ready_requests_queued(), 0); + + for (auto &task_id : to_cancel) { + ASSERT_TRUE(task_manager_.CancelTask(task_id)); + } + + data = std::make_shared(); + task_manager_.FillResourceUsage(false, data); + + resource_load_by_shape = data->resource_load_by_shape(); + shape1 = resource_load_by_shape.resource_demands()[0]; + + ASSERT_EQ(shape1.backlog_size(), 0); + ASSERT_EQ(shape1.num_infeasible_requests_queued(), 0); + ASSERT_EQ(shape1.num_ready_requests_queued(), 0); +} + TEST_F(ClusterTaskManagerTest, OwnerDeadTest) { /* Test the race condition in which the owner of a task dies while the task is pending. diff --git a/src/ray/raylet/scheduling_policy.cc b/src/ray/raylet/scheduling_policy.cc index f0599e11e..d4d2ffc11 100644 --- a/src/ray/raylet/scheduling_policy.cc +++ b/src/ray/raylet/scheduling_policy.cc @@ -151,38 +151,6 @@ std::unordered_map SchedulingPolicy::Schedule( return decision; } -bool SchedulingPolicy::ScheduleBundle( - std::unordered_map &cluster_resources, - const NodeID &local_node_id, const ray::BundleSpecification &bundle_spec) { -#ifndef NDEBUG - RAY_LOG(DEBUG) << "Cluster resource map: "; - for (const auto &node_resource_pair : cluster_resources) { - const NodeID &node_id = node_resource_pair.first; - const SchedulingResources &resources = node_resource_pair.second; - RAY_LOG(DEBUG) << "node_id: " << node_id << " " - << resources.GetAvailableResources().ToString(); - } -#endif - const auto &node_resource_pair = cluster_resources.find(local_node_id); - if (node_resource_pair == cluster_resources.end()) { - return false; - } - const auto &resource_demand = bundle_spec.GetRequiredResources(); - NodeID node_id = node_resource_pair->first; - const auto &node_resources = node_resource_pair->second; - ResourceSet available_node_resources = - ResourceSet(node_resources.GetAvailableResources()); - available_node_resources.SubtractResources(node_resources.GetLoadResources()); - RAY_LOG(DEBUG) << "Scheduling bundle, node id = " << node_id - << ", available resources = " - << node_resources.GetAvailableResources().ToString() - << ", resources load = " << node_resources.GetLoadResources().ToString() - << ", the resource needed = " << resource_demand.ToString(); - /// If the resource_demand is subset of the whole available_node_resources, this bundle - /// can be set in this node, return true. - return resource_demand.IsSubset(available_node_resources); -} - std::vector SchedulingPolicy::SpillOverInfeasibleTasks( SchedulingResources &node_resources) const { // The policy decision to be returned. diff --git a/src/ray/raylet/scheduling_policy.h b/src/ray/raylet/scheduling_policy.h index 24bfdb0cb..f719a1f6e 100644 --- a/src/ray/raylet/scheduling_policy.h +++ b/src/ray/raylet/scheduling_policy.h @@ -50,17 +50,6 @@ class SchedulingPolicy { std::unordered_map &cluster_resources, const NodeID &local_node_id); - /// \param cluster_resources: a set of cluster resources containing resource and load - /// information for some subset of the cluster. - /// \param local_node_id The ID of the node manager that owns this - /// SchedulingPolicy object. - /// \param bundle_spec the description of a bundle which include the resource the bundle - /// need. \return If this bundle can be scheduled in this node, return true; else return - /// false. - bool ScheduleBundle(std::unordered_map &cluster_resources, - const NodeID &local_node_id, - const ray::BundleSpecification &bundle_spec); - /// \brief Given a set of cluster resources, try to spillover infeasible tasks. /// /// \param node_resources The resource information for a node. This may be diff --git a/src/ray/raylet/task_dependency_manager_test.cc b/src/ray/raylet/task_dependency_manager_test.cc index 3f53b5a09..99f6d5622 100644 --- a/src/ray/raylet/task_dependency_manager_test.cc +++ b/src/ray/raylet/task_dependency_manager_test.cc @@ -69,8 +69,8 @@ static inline Task ExampleTask(const std::vector &arguments, FunctionDescriptorBuilder::BuildPython("", "", "", ""), JobID::Nil(), RandomTaskId(), 0, RandomTaskId(), address, num_returns, {}, {}, - std::make_pair(PlacementGroupID::Nil(), -1), true); - builder.SetActorCreationTaskSpec(ActorID::Nil(), 1, {}, 1, false, "", false); + std::make_pair(PlacementGroupID::Nil(), -1), true, ""); + builder.SetActorCreationTaskSpec(ActorID::Nil(), 1, 1, {}, 1, false, "", false); for (const auto &arg : arguments) { builder.AddArg(TaskArgByReference(arg, rpc::Address())); } diff --git a/src/ray/raylet/test/util.h b/src/ray/raylet/test/util.h index 459a31e02..4d64507b8 100644 --- a/src/ray/raylet/test/util.h +++ b/src/ray/raylet/test/util.h @@ -33,9 +33,6 @@ class MockWorker : public WorkerInterface { void AssignTaskId(const TaskID &task_id) {} - // TODO(kfstorm): Remove this once `enable_multi_tenancy` is deleted. - void AssignJobId(const JobID &job_id) {} - void SetAssignedTask(const Task &assigned_task) {} const std::string IpAddress() const { return address_.ip_address(); } diff --git a/src/ray/raylet/worker.cc b/src/ray/raylet/worker.cc index acce4c984..81e8a98ac 100644 --- a/src/ray/raylet/worker.cc +++ b/src/ray/raylet/worker.cc @@ -111,11 +111,6 @@ const std::unordered_set &Worker::GetBlockedTaskIds() const { return blocked_task_ids_; } -void Worker::AssignJobId(const JobID &job_id) { - RAY_CHECK(!RayConfig::instance().enable_multi_tenancy()); - assigned_job_id_ = job_id; -} - const JobID &Worker::GetAssignedJobId() const { return assigned_job_id_; } void Worker::AssignActorId(const ActorID &actor_id) { diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index a40855abe..7cd19868e 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -60,8 +60,6 @@ class WorkerInterface { virtual bool AddBlockedTaskId(const TaskID &task_id) = 0; virtual bool RemoveBlockedTaskId(const TaskID &task_id) = 0; virtual const std::unordered_set &GetBlockedTaskIds() const = 0; - // TODO(kfstorm): Remove this once `enable_multi_tenancy` is deleted. - virtual void AssignJobId(const JobID &job_id) = 0; virtual const JobID &GetAssignedJobId() const = 0; virtual void AssignActorId(const ActorID &actor_id) = 0; virtual const ActorID &GetActorId() const = 0; @@ -151,8 +149,6 @@ class Worker : public WorkerInterface { bool AddBlockedTaskId(const TaskID &task_id); bool RemoveBlockedTaskId(const TaskID &task_id); const std::unordered_set &GetBlockedTaskIds() const; - // TODO(kfstorm): Remove this once `enable_multi_tenancy` is deleted. - void AssignJobId(const JobID &job_id); const JobID &GetAssignedJobId() const; void AssignActorId(const ActorID &actor_id); const ActorID &GetActorId() const; diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index bc649ac02..a7ad5c245 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -54,8 +54,7 @@ namespace ray { namespace raylet { -WorkerPool::WorkerPool(boost::asio::io_service &io_service, int num_workers, - int num_workers_soft_limit, +WorkerPool::WorkerPool(boost::asio::io_service &io_service, int num_workers_soft_limit, int num_initial_python_workers_for_first_job, int maximum_startup_concurrency, int min_worker_port, int max_worker_port, const std::vector &worker_ports, @@ -83,33 +82,7 @@ WorkerPool::WorkerPool(boost::asio::io_service &io_service, int num_workers, for (const auto &entry : worker_commands) { // Initialize the pool state for this language. auto &state = states_by_lang_[entry.first]; - if (!RayConfig::instance().enable_multi_tenancy()) { - switch (entry.first) { - case Language::PYTHON: - state.num_workers_per_process = - RayConfig::instance().num_workers_per_process_python(); - break; - case Language::JAVA: - state.num_workers_per_process = - RayConfig::instance().num_workers_per_process_java(); - break; - case Language::CPP: - state.num_workers_per_process = - RayConfig::instance().num_workers_per_process_cpp(); - break; - default: - RAY_LOG(FATAL) << "The number of workers per process for " - << Language_Name(entry.first) << " worker is not set."; - } - RAY_CHECK(state.num_workers_per_process > 0) - << "Number of workers per process of language " << Language_Name(entry.first) - << " must be positive."; - state.multiple_for_warning = - std::max(state.num_workers_per_process, - std::max(num_workers, maximum_startup_concurrency)); - } else { - state.multiple_for_warning = maximum_startup_concurrency; - } + state.multiple_for_warning = maximum_startup_concurrency; // Set worker command for this language. state.worker_command = entry.second; RAY_CHECK(!state.worker_command.empty()) << "Worker command must not be empty."; @@ -131,27 +104,7 @@ WorkerPool::WorkerPool(boost::asio::io_service &io_service, int num_workers, free_ports_->push(port); } } - if (!RayConfig::instance().enable_multi_tenancy()) { - Start(num_workers); - } else { - ScheduleIdleWorkerKilling(); - } -} - -void WorkerPool::Start(int num_workers) { - RAY_CHECK(!RayConfig::instance().enable_multi_tenancy()); - for (auto &entry : states_by_lang_) { - if (entry.first == Language::JAVA) { - // Disable initial workers for Java. - continue; - } - auto &state = entry.second; - int num_worker_processes = static_cast( - std::ceil(static_cast(num_workers) / state.num_workers_per_process)); - for (int i = 0; i < num_worker_processes; i++) { - StartWorkerProcess(entry.first, rpc::WorkerType::WORKER, JobID::Nil()); - } - } + ScheduleIdleWorkerKilling(); } WorkerPool::~WorkerPool() { @@ -178,7 +131,7 @@ Process WorkerPool::StartWorkerProcess( std::vector dynamic_options, std::unordered_map override_environment_variables) { rpc::JobConfig *job_config = nullptr; - if (RayConfig::instance().enable_multi_tenancy() && !IsIOWorkerType(worker_type)) { + if (!IsIOWorkerType(worker_type)) { RAY_CHECK(!job_id.IsNil()); auto it = all_jobs_.find(job_id); if (it == all_jobs_.end()) { @@ -212,15 +165,12 @@ Process WorkerPool::StartWorkerProcess( int workers_to_start = 1; if (dynamic_options.empty()) { - if (!RayConfig::instance().enable_multi_tenancy()) { - workers_to_start = state.num_workers_per_process; - } else if (language == Language::JAVA) { + if (language == Language::JAVA) { workers_to_start = job_config->num_java_workers_per_process(); } } - // For non-multi-tenancy mode, job code search path is embedded in worker_command. - if (RayConfig::instance().enable_multi_tenancy() && job_config) { + if (job_config) { // Note that we push the item to the front of the vector to make // sure this is the freshest option than others. if (!job_config->jvm_options().empty()) { @@ -272,9 +222,6 @@ Process WorkerPool::StartWorkerProcess( switch (language) { case Language::JAVA: for (auto &entry : raylet_config_) { - if (entry.first == "num_workers_per_process_java") { - continue; - } std::string arg; arg.append("-Dray.raylet.config."); arg.append(entry.first); @@ -282,17 +229,8 @@ Process WorkerPool::StartWorkerProcess( arg.append(entry.second); worker_command_args.push_back(arg); } - if (!RayConfig::instance().enable_multi_tenancy()) { - // The value of `num_workers_per_process_java` may change depends on whether - // dynamic options is empty, so we can't use the value in `RayConfig`. We always - // overwrite the value here. - worker_command_args.push_back( - "-Dray.raylet.config.num_workers_per_process_java=" + - std::to_string(workers_to_start)); - } else { - worker_command_args.push_back("-Dray.job.num-java-workers-per-process=" + - std::to_string(workers_to_start)); - } + worker_command_args.push_back("-Dray.job.num-java-workers-per-process=" + + std::to_string(workers_to_start)); break; default: RAY_LOG(FATAL) @@ -327,12 +265,12 @@ Process WorkerPool::StartWorkerProcess( } ProcessEnvironment env; - if (RayConfig::instance().enable_multi_tenancy() && !IsIOWorkerType(worker_type)) { + if (!IsIOWorkerType(worker_type)) { // We pass the job ID to worker processes via an environment variable, so we don't // need to add a new CLI parameter for both Python and Java workers. env.emplace(kEnvVarKeyJobId, job_id.Hex()); } - if (RayConfig::instance().enable_multi_tenancy() && job_config) { + if (job_config) { env.insert(job_config->worker_env().begin(), job_config->worker_env().end()); } @@ -516,18 +454,16 @@ void WorkerPool::OnWorkerStarted(const std::shared_ptr &worker) io_worker_state.num_starting_io_workers--; } - if (RayConfig::instance().enable_multi_tenancy()) { - // This is a workaround to finish driver registration after all initial workers are - // registered to Raylet if and only if Raylet is started by a Python driver and the - // job config is not set in `ray.init(...)`. - if (first_job_ == worker->GetAssignedJobId() && - worker->GetLanguage() == Language::PYTHON) { - if (++first_job_registered_python_worker_count_ == - first_job_driver_wait_num_python_workers_) { - if (first_job_send_register_client_reply_to_driver_) { - first_job_send_register_client_reply_to_driver_(); - first_job_send_register_client_reply_to_driver_ = nullptr; - } + // This is a workaround to finish driver registration after all initial workers are + // registered to Raylet if and only if Raylet is started by a Python driver and the + // job config is not set in `ray.init(...)`. + if (first_job_ == worker->GetAssignedJobId() && + worker->GetLanguage() == Language::PYTHON) { + if (++first_job_registered_python_worker_count_ == + first_job_driver_wait_num_python_workers_) { + if (first_job_send_register_client_reply_to_driver_) { + first_job_send_register_client_reply_to_driver_(); + first_job_send_register_client_reply_to_driver_ = nullptr; } } } @@ -554,18 +490,15 @@ Status WorkerPool::RegisterDriver(const std::shared_ptr &driver // Invoke the `send_reply_callback` later to only finish driver // registration after all initial workers are registered to Raylet. bool delay_callback = false; - // Multi-tenancy is enabled. - if (RayConfig().instance().enable_multi_tenancy()) { - // If this is the first job. - if (first_job_.IsNil()) { - first_job_ = job_id; - // If the number of Python workers we need to wait is positive. - if (num_initial_python_workers_for_first_job_ > 0) { - delay_callback = true; - // Start initial Python workers for the first job. - for (int i = 0; i < num_initial_python_workers_for_first_job_; i++) { - StartWorkerProcess(Language::PYTHON, rpc::WorkerType::WORKER, job_id); - } + // If this is the first job. + if (first_job_.IsNil()) { + first_job_ = job_id; + // If the number of Python workers we need to wait is positive. + if (num_initial_python_workers_for_first_job_ > 0) { + delay_callback = true; + // Start initial Python workers for the first job. + for (int i = 0; i < num_initial_python_workers_for_first_job_; i++) { + StartWorkerProcess(Language::PYTHON, rpc::WorkerType::WORKER, job_id); } } } @@ -695,11 +628,9 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { // Put the worker to the corresponding idle pool. if (worker->GetActorId().IsNil()) { state.idle.insert(worker); - if (RayConfig::instance().enable_multi_tenancy()) { - int64_t now = current_time_ms(); - idle_of_all_languages_.emplace_back(worker, now); - idle_of_all_languages_map_[worker] = now; - } + int64_t now = current_time_ms(); + idle_of_all_languages_.emplace_back(worker, now); + idle_of_all_languages_map_[worker] = now; } else { state.idle_actor[worker->GetActorId()] = worker; } @@ -783,11 +714,11 @@ void WorkerPool::TryKillingIdleWorkers() { } for (const auto &worker : workers_in_the_same_process) { - RAY_LOG(INFO) << "The worker pool has " << running_size - << " registered workers which exceeds the soft limit of " - << num_workers_soft_limit_ << ", and worker " << worker->WorkerId() - << " with pid " << process.GetId() - << " has been idle for a a while. Kill it."; + RAY_LOG(DEBUG) << "The worker pool has " << running_size + << " registered workers which exceeds the soft limit of " + << num_workers_soft_limit_ << ", and worker " << worker->WorkerId() + << " with pid " << process.GetId() + << " has been idle for a a while. Kill it."; // To avoid object lost issue caused by forcibly killing, send an RPC request to the // worker to allow it to do cleanup before exiting. auto rpc_client = worker->rpc_client(); @@ -858,40 +789,28 @@ std::shared_ptr WorkerPool::PopWorker( } } else if (!task_spec.IsActorTask()) { // Code path of normal task or actor creation task without dynamic worker options. - if (!RayConfig::instance().enable_multi_tenancy()) { - if (!state.idle.empty()) { - worker = std::move(*state.idle.begin()); - state.idle.erase(state.idle.begin()); - } else { - // There are no more non-actor workers available to execute this task. - // Start a new worker process. - proc = StartWorkerProcess(task_spec.GetLanguage(), rpc::WorkerType::WORKER, - JobID::Nil()); - } - } else { - // Find an available worker which is already assigned to this job. - // Try to pop the most recently pushed worker. - for (auto it = idle_of_all_languages_.rbegin(); it != idle_of_all_languages_.rend(); - it++) { - if (task_spec.GetLanguage() != it->first->GetLanguage() || - it->first->GetAssignedJobId() != task_spec.JobId()) { - continue; - } - state.idle.erase(it->first); - // We can't erase a reverse_iterator. - auto lit = it.base(); - lit--; - worker = std::move(lit->first); - idle_of_all_languages_.erase(lit); - idle_of_all_languages_map_.erase(worker); - break; - } - if (worker == nullptr) { - // There are no more non-actor workers available to execute this task. - // Start a new worker process. - proc = StartWorkerProcess(task_spec.GetLanguage(), rpc::WorkerType::WORKER, - task_spec.JobId()); + // Find an available worker which is already assigned to this job. + // Try to pop the most recently pushed worker. + for (auto it = idle_of_all_languages_.rbegin(); it != idle_of_all_languages_.rend(); + it++) { + if (task_spec.GetLanguage() != it->first->GetLanguage() || + it->first->GetAssignedJobId() != task_spec.JobId()) { + continue; } + state.idle.erase(it->first); + // We can't erase a reverse_iterator. + auto lit = it.base(); + lit--; + worker = std::move(lit->first); + idle_of_all_languages_.erase(lit); + idle_of_all_languages_map_.erase(worker); + break; + } + if (worker == nullptr) { + // There are no more non-actor workers available to execute this task. + // Start a new worker process. + proc = StartWorkerProcess(task_spec.GetLanguage(), rpc::WorkerType::WORKER, + task_spec.JobId()); } } else { // Code path of actor task. @@ -907,7 +826,7 @@ std::shared_ptr WorkerPool::PopWorker( WarnAboutSize(); } - if (RayConfig::instance().enable_multi_tenancy() && worker) { + if (worker) { RAY_CHECK(worker->GetAssignedJobId() == task_spec.JobId()); } return worker; diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 78ae85564..62dfd20a5 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -95,7 +95,6 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { /// process should create and register the specified number of workers, and add them to /// the pool. /// - /// \param num_workers The number of workers to start, per language. /// \param num_workers_soft_limit The soft limit of the number of workers. /// \param num_initial_python_workers_for_first_job The number of initial Python /// workers for the first job. @@ -113,8 +112,8 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { /// \param raylet_config The raylet config list of this node. /// \param starting_worker_timeout_callback The callback that will be triggered once /// it times out to start a worker. - WorkerPool(boost::asio::io_service &io_service, int num_workers, - int num_workers_soft_limit, int num_initial_python_workers_for_first_job, + WorkerPool(boost::asio::io_service &io_service, int num_workers_soft_limit, + int num_initial_python_workers_for_first_job, int maximum_startup_concurrency, int min_worker_port, int max_worker_port, const std::vector &worker_ports, std::shared_ptr gcs_client, @@ -251,7 +250,7 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { /// Try to prestart a number of workers suitable the given task spec. Prestarting /// is needed since core workers request one lease at a time, if starting is slow, - /// then it means it takes a long time to scale up when multi-tenancy is on. + /// then it means it takes a long time to scale up. /// /// \param task_spec The returned worker must be able to execute this task. /// \param backlog_size The number of tasks in the client backlog of this shape. @@ -306,7 +305,7 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { protected: /// Asynchronously start a new worker process. Once the worker process has /// registered with an external server, the process should create and - /// register num_workers_per_process workers, then add them to the pool. + /// register N workers, then add them to the pool. /// Failure to start the worker process is a fatal error. If too many workers /// are already being started, then this function will return without starting /// any workers. @@ -354,8 +353,6 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { struct State { /// The commands and arguments used to start the worker process std::vector worker_command; - /// The number of workers per process. - int num_workers_per_process; /// The pool of dedicated workers for actor creation tasks /// with prefix or suffix worker command. std::unordered_map> idle_dedicated_workers; @@ -392,12 +389,6 @@ class WorkerPool : public WorkerPoolInterface, public IOWorkerPoolInterface { std::unordered_map> states_by_lang_; private: - /// Force-start at least num_workers workers for this language. Used for internal and - /// test purpose only. - /// - /// \param num_workers The number of workers to start, per language. - void Start(int num_workers); - /// A helper function that returns the reference of the pool state /// for a given language. State &GetStateForLanguage(const Language &language); diff --git a/src/ray/raylet/worker_pool_test.cc b/src/ray/raylet/worker_pool_test.cc index 833be632b..ee8f3356b 100644 --- a/src/ray/raylet/worker_pool_test.cc +++ b/src/ray/raylet/worker_pool_test.cc @@ -36,12 +36,9 @@ class WorkerPoolMock : public WorkerPool { public: explicit WorkerPoolMock(boost::asio::io_service &io_service, const WorkerCommandMap &worker_commands) - : WorkerPool(io_service, 0, POOL_SIZE_SOFT_LIMIT, 0, MAXIMUM_STARTUP_CONCURRENCY, 0, - 0, {}, nullptr, worker_commands, {}, []() {}), - last_worker_process_() { - states_by_lang_[ray::Language::JAVA].num_workers_per_process = - NUM_WORKERS_PER_PROCESS_JAVA; - } + : WorkerPool(io_service, POOL_SIZE_SOFT_LIMIT, 0, MAXIMUM_STARTUP_CONCURRENCY, 0, 0, + {}, nullptr, worker_commands, {}, []() {}), + last_worker_process_() {} ~WorkerPoolMock() { // Avoid killing real processes @@ -103,14 +100,11 @@ class WorkerPoolMock : public WorkerPool { std::unordered_map> worker_commands_by_proc_; }; -class WorkerPoolTest : public ::testing::TestWithParam { +class WorkerPoolTest : public ::testing::Test { public: WorkerPoolTest() : error_message_type_(1), client_call_manager_(io_service_) { - bool enable_multi_tenancy = GetParam(); RayConfig::instance().initialize( - {{"enable_multi_tenancy", std::to_string(enable_multi_tenancy)}, - {"num_workers_per_process_java", std::to_string(NUM_WORKERS_PER_PROCESS_JAVA)}, - {"object_spilling_config", "mock_config"}, + {{"object_spilling_config", "mock_config"}, {"max_io_workers", std::to_string(MAX_IO_WORKER_SIZE)}}); SetWorkerCommands( {{Language::PYTHON, {"dummy_py_worker_command"}}, @@ -236,16 +230,10 @@ static inline TaskSpecification ExampleTaskSpec( } static inline std::string GetNumJavaWorkersPerProcessSystemProperty(int num) { - std::string key; - if (RayConfig::instance().enable_multi_tenancy()) { - key = "ray.job.num-java-workers-per-process"; - } else { - key = "ray.raylet.config.num_workers_per_process_java"; - } - return std::string("-D") + key + "=" + std::to_string(num); + return std::string("-Dray.job.num-java-workers-per-process=") + std::to_string(num); } -TEST_P(WorkerPoolTest, CompareWorkerProcessObjects) { +TEST_F(WorkerPoolTest, CompareWorkerProcessObjects) { typedef Process T; T a(T::CreateNewDummy()), b(T::CreateNewDummy()), empty = T(); ASSERT_TRUE(empty.IsNull()); @@ -259,7 +247,7 @@ TEST_P(WorkerPoolTest, CompareWorkerProcessObjects) { ASSERT_TRUE(!std::equal_to()(a, empty)); } -TEST_P(WorkerPoolTest, HandleWorkerRegistration) { +TEST_F(WorkerPoolTest, HandleWorkerRegistration) { Process proc = worker_pool_->StartWorkerProcess(Language::JAVA, rpc::WorkerType::WORKER, JOB_ID); std::vector> workers; @@ -286,59 +274,49 @@ TEST_P(WorkerPoolTest, HandleWorkerRegistration) { } } -TEST_P(WorkerPoolTest, HandleUnknownWorkerRegistration) { +TEST_F(WorkerPoolTest, HandleUnknownWorkerRegistration) { auto worker = CreateWorker(Process(), Language::PYTHON); auto status = worker_pool_->RegisterWorker(worker, 1234, [](Status, int) {}); ASSERT_FALSE(status.ok()); } -TEST_P(WorkerPoolTest, StartupPythonWorkerProcessCount) { +TEST_F(WorkerPoolTest, StartupPythonWorkerProcessCount) { TestStartupWorkerProcessCount(Language::PYTHON, 1, {"dummy_py_worker_command"}); } -TEST_P(WorkerPoolTest, StartupJavaWorkerProcessCount) { +TEST_F(WorkerPoolTest, StartupJavaWorkerProcessCount) { TestStartupWorkerProcessCount( Language::JAVA, NUM_WORKERS_PER_PROCESS_JAVA, {"dummy_java_worker_command", GetNumJavaWorkersPerProcessSystemProperty(NUM_WORKERS_PER_PROCESS_JAVA)}); } -TEST_P(WorkerPoolTest, InitialWorkerProcessCount) { - if (!RayConfig::instance().enable_multi_tenancy()) { - worker_pool_->Start(1); - // Here we try to start only 1 worker for each worker language. But since we disabled - // initial workers for Java, we expect to see only 1 worker which is a Python worker. - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 1); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 1); - } else { - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 0); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 0); - } +TEST_F(WorkerPoolTest, InitialWorkerProcessCount) { + ASSERT_EQ(worker_pool_->NumWorkersStarting(), 0); + ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 0); } -TEST_P(WorkerPoolTest, TestPrestartingWorkers) { - if (RayConfig::instance().enable_multi_tenancy()) { - const auto task_spec = ExampleTaskSpec(); - // Prestarts 2 workers. - worker_pool_->PrestartWorkers(task_spec, 2); - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 2); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 2); - // Prestarts 1 more worker. - worker_pool_->PrestartWorkers(task_spec, 3); - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 3); - // No more needed. - worker_pool_->PrestartWorkers(task_spec, 1); - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 3); - // Capped by soft limit of 5. - worker_pool_->PrestartWorkers(task_spec, 20); - ASSERT_EQ(worker_pool_->NumWorkersStarting(), 5); - ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 5); - } +TEST_F(WorkerPoolTest, TestPrestartingWorkers) { + const auto task_spec = ExampleTaskSpec(); + // Prestarts 2 workers. + worker_pool_->PrestartWorkers(task_spec, 2); + ASSERT_EQ(worker_pool_->NumWorkersStarting(), 2); + ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 2); + // Prestarts 1 more worker. + worker_pool_->PrestartWorkers(task_spec, 3); + ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); + ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 3); + // No more needed. + worker_pool_->PrestartWorkers(task_spec, 1); + ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); + ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 3); + // Capped by soft limit of 5. + worker_pool_->PrestartWorkers(task_spec, 20); + ASSERT_EQ(worker_pool_->NumWorkersStarting(), 5); + ASSERT_EQ(worker_pool_->NumWorkerProcessesStarting(), 5); } -TEST_P(WorkerPoolTest, HandleWorkerPushPop) { +TEST_F(WorkerPoolTest, HandleWorkerPushPop) { // Try to pop a worker from the empty pool and make sure we don't get one. std::shared_ptr popped_worker; const auto task_spec = ExampleTaskSpec(); @@ -365,7 +343,7 @@ TEST_P(WorkerPoolTest, HandleWorkerPushPop) { ASSERT_EQ(popped_worker, nullptr); } -TEST_P(WorkerPoolTest, PopActorWorker) { +TEST_F(WorkerPoolTest, PopActorWorker) { // Create a worker. auto worker = CreateWorker(Process::CreateNewDummy()); // Add the worker to the pool. @@ -387,7 +365,7 @@ TEST_P(WorkerPoolTest, PopActorWorker) { ASSERT_EQ(actor->GetActorId(), actor_id); } -TEST_P(WorkerPoolTest, PopWorkersOfMultipleLanguages) { +TEST_F(WorkerPoolTest, PopWorkersOfMultipleLanguages) { // Create a Python Worker, and add it to the pool auto py_worker = CreateWorker(Process::CreateNewDummy(), Language::PYTHON); worker_pool_->PushWorker(py_worker); @@ -405,7 +383,7 @@ TEST_P(WorkerPoolTest, PopWorkersOfMultipleLanguages) { ASSERT_NE(worker_pool_->PopWorker(java_task_spec), nullptr); } -TEST_P(WorkerPoolTest, StartWorkerWithDynamicOptionsCommand) { +TEST_F(WorkerPoolTest, StartWorkerWithDynamicOptionsCommand) { const std::vector java_worker_command = { "RAY_WORKER_DYNAMIC_OPTION_PLACEHOLDER", "dummy_java_worker_command", "RAY_WORKER_RAYLET_CONFIG_PLACEHOLDER"}; @@ -426,25 +404,15 @@ TEST_P(WorkerPoolTest, StartWorkerWithDynamicOptionsCommand) { const auto real_command = worker_pool_->GetWorkerCommand(worker_pool_->LastStartedWorkerProcess()); - if (RayConfig::instance().enable_multi_tenancy()) { - ASSERT_EQ( - real_command, - std::vector( - {"test_op_0", "test_op_1", "-Dray.job.code-search-path=/test/code_serch_path", - "dummy_java_worker_command", GetNumJavaWorkersPerProcessSystemProperty(1)})); - } else { - ASSERT_EQ(real_command, std::vector( - {"test_op_0", "test_op_1", "dummy_java_worker_command", - GetNumJavaWorkersPerProcessSystemProperty(1)})); - } + ASSERT_EQ( + real_command, + std::vector( + {"test_op_0", "test_op_1", "-Dray.job.code-search-path=/test/code_serch_path", + "dummy_java_worker_command", GetNumJavaWorkersPerProcessSystemProperty(1)})); worker_pool_->HandleJobFinished(JOB_ID); } -TEST_P(WorkerPoolTest, PopWorkerMultiTenancy) { - if (!RayConfig::instance().enable_multi_tenancy()) { - return; - } - +TEST_F(WorkerPoolTest, PopWorkerMultiTenancy) { auto job_id1 = JOB_ID; auto job_id2 = JobID::FromInt(2); ASSERT_NE(job_id1, job_id2); @@ -505,7 +473,7 @@ TEST_P(WorkerPoolTest, PopWorkerMultiTenancy) { } } -TEST_P(WorkerPoolTest, MaximumStartupConcurrency) { +TEST_F(WorkerPoolTest, MaximumStartupConcurrency) { auto task_spec = ExampleTaskSpec(); std::vector started_processes; @@ -550,7 +518,7 @@ TEST_P(WorkerPoolTest, MaximumStartupConcurrency) { ASSERT_EQ(0, worker_pool_->NumWorkerProcessesStarting()); } -TEST_P(WorkerPoolTest, HandleIOWorkersPushPop) { +TEST_F(WorkerPoolTest, HandleIOWorkersPushPop) { std::unordered_set> spill_pushed_worker; std::unordered_set> restore_pushed_worker; auto spill_worker_callback = @@ -603,7 +571,7 @@ TEST_P(WorkerPoolTest, HandleIOWorkersPushPop) { ASSERT_EQ(restore_pushed_worker.size(), 1); } -TEST_P(WorkerPoolTest, MaxIOWorkerSimpleTest) { +TEST_F(WorkerPoolTest, MaxIOWorkerSimpleTest) { // Make sure max number of spill workers are respected. auto callback = [](std::shared_ptr worker) {}; std::vector started_processes; @@ -633,7 +601,7 @@ TEST_P(WorkerPoolTest, MaxIOWorkerSimpleTest) { ASSERT_EQ(worker_pool_->NumSpillWorkerStarting(), 0); } -TEST_P(WorkerPoolTest, MaxIOWorkerComplicateTest) { +TEST_F(WorkerPoolTest, MaxIOWorkerComplicateTest) { // Make sure max number of restore workers are respected. // This test will test a little more complicated scneario. // For example, it tests scenarios where there are @@ -685,7 +653,7 @@ TEST_P(WorkerPoolTest, MaxIOWorkerComplicateTest) { ASSERT_EQ(worker_pool_->NumSpillWorkerStarting(), 0); } -TEST_P(WorkerPoolTest, MaxSpillRestoreWorkersIntegrationTest) { +TEST_F(WorkerPoolTest, MaxSpillRestoreWorkersIntegrationTest) { auto callback = [](std::shared_ptr worker) {}; // Run many pop spill/restore workers and make sure the max worker size doesn't exceed. std::vector started_restore_processes; @@ -732,7 +700,7 @@ TEST_P(WorkerPoolTest, MaxSpillRestoreWorkersIntegrationTest) { ASSERT_EQ(worker_pool_->GetProcessSize(), 2 * MAX_IO_WORKER_SIZE); } -TEST_P(WorkerPoolTest, DeleteWorkerPushPop) { +TEST_F(WorkerPoolTest, DeleteWorkerPushPop) { /// Make sure delete workers always pop an I/O worker that has more idle worker in their /// pools. // 2 spill worker and 1 restore worker. @@ -770,9 +738,6 @@ TEST_P(WorkerPoolTest, DeleteWorkerPushPop) { }); } -INSTANTIATE_TEST_CASE_P(WorkerPoolMultiTenancyTest, WorkerPoolTest, - ::testing::Values(true, false)); - } // namespace raylet } // namespace ray diff --git a/src/ray/raylet_client/raylet_client.h b/src/ray/raylet_client/raylet_client.h index a7fa8dcb0..a50b7c0e7 100644 --- a/src/ray/raylet_client/raylet_client.h +++ b/src/ray/raylet_client/raylet_client.h @@ -139,6 +139,14 @@ class DependencyWaiterInterface { virtual ~DependencyWaiterInterface(){}; }; +class RayletClientInterface : public PinObjectsInterface, + public WorkerLeaseInterface, + public DependencyWaiterInterface, + public ResourceReserveInterface { + public: + virtual ~RayletClientInterface(){}; +}; + namespace raylet { class RayletConnection { @@ -171,10 +179,7 @@ class RayletConnection { std::mutex write_mutex_; }; -class RayletClient : public PinObjectsInterface, - public WorkerLeaseInterface, - public DependencyWaiterInterface, - public ResourceReserveInterface { +class RayletClient : public RayletClientInterface { public: /// Connect to the raylet. /// diff --git a/src/ray/rpc/gcs_server/gcs_rpc_client.h b/src/ray/rpc/gcs_server/gcs_rpc_client.h index faba25e01..82857123e 100644 --- a/src/ray/rpc/gcs_server/gcs_rpc_client.h +++ b/src/ray/rpc/gcs_server/gcs_rpc_client.h @@ -97,6 +97,12 @@ class GcsRpcClient { new GrpcClient(address, port, client_call_manager)); node_info_grpc_client_ = std::unique_ptr>( new GrpcClient(address, port, client_call_manager)); + node_resource_info_grpc_client_ = + std::unique_ptr>( + new GrpcClient(address, port, + client_call_manager)); + heartbeat_info_grpc_client_ = std::unique_ptr>( + new GrpcClient(address, port, client_call_manager)); object_info_grpc_client_ = std::unique_ptr>( new GrpcClient(address, port, client_call_manager)); task_info_grpc_client_ = std::unique_ptr>( @@ -155,15 +161,6 @@ class GcsRpcClient { /// Get information of all nodes from GCS Service. VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetAllNodeInfo, node_info_grpc_client_, ) - /// Report heartbeat of a node to GCS Service. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, ReportHeartbeat, - node_info_grpc_client_, ) - - /// Get newest heartbeat of all nodes from GCS Service. Only used when light heartbeat - /// enabled. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetAllHeartbeat, - node_info_grpc_client_, ) - /// Report resource usage of a node to GCS Service. VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, ReportResourceUsage, node_info_grpc_client_, ) @@ -172,17 +169,6 @@ class GcsRpcClient { VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetAllResourceUsage, node_info_grpc_client_, ) - /// Get node's resources from GCS Service. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetResources, node_info_grpc_client_, ) - - /// Update resources of a node in GCS Service. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, UpdateResources, - node_info_grpc_client_, ) - - /// Delete resources of a node in GCS Service. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, DeleteResources, - node_info_grpc_client_, ) - /// Set internal config of the cluster in the GCS Service. VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, SetInternalConfig, node_info_grpc_client_, ) @@ -191,9 +177,25 @@ class GcsRpcClient { VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetInternalConfig, node_info_grpc_client_, ) + /// Get node's resources from GCS Service. + VOID_GCS_RPC_CLIENT_METHOD(NodeResourceInfoGcsService, GetResources, + node_resource_info_grpc_client_, ) + + /// Update resources of a node in GCS Service. + VOID_GCS_RPC_CLIENT_METHOD(NodeResourceInfoGcsService, UpdateResources, + node_resource_info_grpc_client_, ) + + /// Delete resources of a node in GCS Service. + VOID_GCS_RPC_CLIENT_METHOD(NodeResourceInfoGcsService, DeleteResources, + node_resource_info_grpc_client_, ) + /// Get available resources of all nodes from the GCS Service. - VOID_GCS_RPC_CLIENT_METHOD(NodeInfoGcsService, GetAllAvailableResources, - node_info_grpc_client_, ) + VOID_GCS_RPC_CLIENT_METHOD(NodeResourceInfoGcsService, GetAllAvailableResources, + node_resource_info_grpc_client_, ) + + /// Report heartbeat of a node to GCS Service. + VOID_GCS_RPC_CLIENT_METHOD(HeartbeatInfoGcsService, ReportHeartbeat, + heartbeat_info_grpc_client_, ) /// Get object's locations from GCS Service. VOID_GCS_RPC_CLIENT_METHOD(ObjectInfoGcsService, GetObjectLocations, @@ -267,6 +269,10 @@ class GcsRpcClient { VOID_GCS_RPC_CLIENT_METHOD(PlacementGroupInfoGcsService, GetAllPlacementGroup, placement_group_info_grpc_client_, ) + /// Wait for placement group until ready via GCS Service. + VOID_GCS_RPC_CLIENT_METHOD(PlacementGroupInfoGcsService, WaitPlacementGroupUntilReady, + placement_group_info_grpc_client_, ) + private: std::function gcs_service_failure_detected_; @@ -274,6 +280,8 @@ class GcsRpcClient { std::unique_ptr> job_info_grpc_client_; std::unique_ptr> actor_info_grpc_client_; std::unique_ptr> node_info_grpc_client_; + std::unique_ptr> node_resource_info_grpc_client_; + std::unique_ptr> heartbeat_info_grpc_client_; std::unique_ptr> object_info_grpc_client_; std::unique_ptr> task_info_grpc_client_; std::unique_ptr> stats_grpc_client_; diff --git a/src/ray/rpc/gcs_server/gcs_rpc_server.h b/src/ray/rpc/gcs_server/gcs_rpc_server.h index aecabcfa2..248ec9837 100644 --- a/src/ray/rpc/gcs_server/gcs_rpc_server.h +++ b/src/ray/rpc/gcs_server/gcs_rpc_server.h @@ -30,6 +30,12 @@ namespace rpc { #define NODE_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(NodeInfoGcsService, HANDLER) +#define HEARTBEAT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(HeartbeatInfoGcsService, HANDLER) + +#define NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(NodeResourceInfoGcsService, HANDLER) + #define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(ObjectInfoGcsService, HANDLER) @@ -177,14 +183,6 @@ class NodeInfoGcsServiceHandler { GetAllNodeInfoReply *reply, SendReplyCallback send_reply_callback) = 0; - virtual void HandleReportHeartbeat(const ReportHeartbeatRequest &request, - ReportHeartbeatReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllHeartbeat(const GetAllHeartbeatRequest &request, - GetAllHeartbeatReply *reply, - SendReplyCallback send_reply_callback) = 0; - virtual void HandleReportResourceUsage(const ReportResourceUsageRequest &request, ReportResourceUsageReply *reply, SendReplyCallback send_reply_callback) = 0; @@ -193,18 +191,6 @@ class NodeInfoGcsServiceHandler { GetAllResourceUsageReply *reply, SendReplyCallback send_reply_callback) = 0; - virtual void HandleGetResources(const GetResourcesRequest &request, - GetResourcesReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleUpdateResources(const UpdateResourcesRequest &request, - UpdateResourcesReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleDeleteResources(const DeleteResourcesRequest &request, - DeleteResourcesReply *reply, - SendReplyCallback send_reply_callback) = 0; - virtual void HandleSetInternalConfig(const SetInternalConfigRequest &request, SetInternalConfigReply *reply, SendReplyCallback send_reply_callback) = 0; @@ -212,11 +198,6 @@ class NodeInfoGcsServiceHandler { virtual void HandleGetInternalConfig(const GetInternalConfigRequest &request, GetInternalConfigReply *reply, SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllAvailableResources( - const rpc::GetAllAvailableResourcesRequest &request, - rpc::GetAllAvailableResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; }; /// The `GrpcService` for `NodeInfoGcsService`. @@ -238,16 +219,10 @@ class NodeInfoGrpcService : public GrpcService { NODE_INFO_SERVICE_RPC_HANDLER(RegisterNode); NODE_INFO_SERVICE_RPC_HANDLER(UnregisterNode); NODE_INFO_SERVICE_RPC_HANDLER(GetAllNodeInfo); - NODE_INFO_SERVICE_RPC_HANDLER(ReportHeartbeat); - NODE_INFO_SERVICE_RPC_HANDLER(GetAllHeartbeat); NODE_INFO_SERVICE_RPC_HANDLER(ReportResourceUsage); NODE_INFO_SERVICE_RPC_HANDLER(GetAllResourceUsage); - NODE_INFO_SERVICE_RPC_HANDLER(GetResources); - NODE_INFO_SERVICE_RPC_HANDLER(UpdateResources); - NODE_INFO_SERVICE_RPC_HANDLER(DeleteResources); NODE_INFO_SERVICE_RPC_HANDLER(SetInternalConfig); NODE_INFO_SERVICE_RPC_HANDLER(GetInternalConfig); - NODE_INFO_SERVICE_RPC_HANDLER(GetAllAvailableResources); } private: @@ -257,6 +232,89 @@ class NodeInfoGrpcService : public GrpcService { NodeInfoGcsServiceHandler &service_handler_; }; +class NodeResourceInfoGcsServiceHandler { + public: + virtual ~NodeResourceInfoGcsServiceHandler() = default; + + virtual void HandleGetResources(const GetResourcesRequest &request, + GetResourcesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleUpdateResources(const UpdateResourcesRequest &request, + UpdateResourcesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleDeleteResources(const DeleteResourcesRequest &request, + DeleteResourcesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllAvailableResources( + const rpc::GetAllAvailableResourcesRequest &request, + rpc::GetAllAvailableResourcesReply *reply, + rpc::SendReplyCallback send_reply_callback) = 0; +}; + +/// The `GrpcService` for `NodeResourceInfoGcsService`. +class NodeResourceInfoGrpcService : public GrpcService { + public: + /// Constructor. + /// + /// \param[in] handler The service handler that actually handle the requests. + explicit NodeResourceInfoGrpcService(boost::asio::io_service &io_service, + NodeResourceInfoGcsServiceHandler &handler) + : GrpcService(io_service), service_handler_(handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories) override { + NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetResources); + NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(UpdateResources); + NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(DeleteResources); + NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetAllAvailableResources); + } + + private: + /// The grpc async service object. + NodeResourceInfoGcsService::AsyncService service_; + /// The service handler that actually handle the requests. + NodeResourceInfoGcsServiceHandler &service_handler_; +}; + +class HeartbeatInfoGcsServiceHandler { + public: + virtual ~HeartbeatInfoGcsServiceHandler() = default; + virtual void HandleReportHeartbeat(const ReportHeartbeatRequest &request, + ReportHeartbeatReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; +/// The `GrpcService` for `HeartbeatInfoGcsService`. +class HeartbeatInfoGrpcService : public GrpcService { + public: + /// Constructor. + /// + /// \param[in] handler The service handler that actually handle the requests. + explicit HeartbeatInfoGrpcService(boost::asio::io_service &io_service, + HeartbeatInfoGcsServiceHandler &handler) + : GrpcService(io_service), service_handler_(handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories) override { + HEARTBEAT_INFO_SERVICE_RPC_HANDLER(ReportHeartbeat); + } + + private: + /// The grpc async service object. + HeartbeatInfoGcsService::AsyncService service_; + /// The service handler that actually handle the requests. + HeartbeatInfoGcsServiceHandler &service_handler_; +}; + class ObjectInfoGcsServiceHandler { public: virtual ~ObjectInfoGcsServiceHandler() = default; @@ -474,6 +532,11 @@ class PlacementGroupInfoGcsServiceHandler { virtual void HandleGetAllPlacementGroup(const GetAllPlacementGroupRequest &request, GetAllPlacementGroupReply *reply, SendReplyCallback send_reply_callback) = 0; + + virtual void HandleWaitPlacementGroupUntilReady( + const WaitPlacementGroupUntilReadyRequest &request, + WaitPlacementGroupUntilReadyReply *reply, + SendReplyCallback send_reply_callback) = 0; }; /// The `GrpcService` for `PlacementGroupInfoGcsService`. @@ -496,6 +559,7 @@ class PlacementGroupInfoGrpcService : public GrpcService { PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(RemovePlacementGroup); PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(GetPlacementGroup); PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(GetAllPlacementGroup); + PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(WaitPlacementGroupUntilReady); } private: @@ -508,6 +572,8 @@ class PlacementGroupInfoGrpcService : public GrpcService { using JobInfoHandler = JobInfoGcsServiceHandler; using ActorInfoHandler = ActorInfoGcsServiceHandler; using NodeInfoHandler = NodeInfoGcsServiceHandler; +using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; +using HeartbeatInfoHandler = HeartbeatInfoGcsServiceHandler; using ObjectInfoHandler = ObjectInfoGcsServiceHandler; using TaskInfoHandler = TaskInfoGcsServiceHandler; using StatsHandler = StatsGcsServiceHandler; diff --git a/src/ray/rpc/node_manager/node_manager_client_pool.cc b/src/ray/rpc/node_manager/node_manager_client_pool.cc new file mode 100644 index 000000000..afc22987c --- /dev/null +++ b/src/ray/rpc/node_manager/node_manager_client_pool.cc @@ -0,0 +1,42 @@ +#include "ray/rpc/node_manager/node_manager_client_pool.h" + +namespace ray { +namespace rpc { + +shared_ptr NodeManagerClientPool::GetOrConnectByAddress( + const rpc::Address &address) { + RAY_CHECK(address.raylet_id() != ""); + absl::MutexLock lock(&mu_); + auto raylet_id = NodeID::FromBinary(address.raylet_id()); + auto it = client_map_.find(raylet_id); + if (it != client_map_.end()) { + return it->second; + } + auto connection = client_factory_(address); + client_map_[raylet_id] = connection; + + RAY_LOG(DEBUG) << "Connected to " << address.ip_address() << ":" << address.port(); + return connection; +} + +optional> NodeManagerClientPool::GetOrConnectByID( + ray::NodeID id) { + absl::MutexLock lock(&mu_); + auto it = client_map_.find(id); + if (it == client_map_.end()) { + return {}; + } + return it->second; +} + +void NodeManagerClientPool::Disconnect(ray::NodeID id) { + absl::MutexLock lock(&mu_); + auto it = client_map_.find(id); + if (it == client_map_.end()) { + return; + } + client_map_.erase(it); +} + +} // namespace rpc +} // namespace ray diff --git a/src/ray/rpc/node_manager/node_manager_client_pool.h b/src/ray/rpc/node_manager/node_manager_client_pool.h new file mode 100644 index 000000000..071b8519a --- /dev/null +++ b/src/ray/rpc/node_manager/node_manager_client_pool.h @@ -0,0 +1,84 @@ +// Copyright 2020 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" +#include "ray/common/id.h" +#include "ray/raylet_client/raylet_client.h" +#include "ray/rpc/node_manager/node_manager_client.h" + +using absl::optional; +using std::shared_ptr; + +namespace ray { +namespace rpc { + +using RayletClientFactoryFn = + std::function(const rpc::Address &)>; +class NodeManagerClientPool { + public: + NodeManagerClientPool() = delete; + + /// Return an existing NodeManagerWorkerClient if exists, and connect to one if it does + /// not. The returned pointer is borrowed, and expected to be used briefly. + optional> GetOrConnectByID(ray::NodeID id); + + /// Return an existing NodeManagerWorkerClient if exists, and connect to one if it does + /// not. The returned pointer is borrowed, and expected to be used briefly. + shared_ptr GetOrConnectByAddress( + const rpc::Address &address); + + /// Removes a connection to the worker from the pool, if one exists. Since the + /// shared pointer will no longer be retained in the pool, the connection will + /// be open until it's no longer used, at which time it will disconnect. + void Disconnect(ray::NodeID id); + + NodeManagerClientPool(rpc::ClientCallManager &ccm) + : client_factory_(defaultClientFactory(ccm)){}; + + NodeManagerClientPool(RayletClientFactoryFn client_factory) + : client_factory_(client_factory){}; + + private: + /// Provides the default client factory function. Providing this function to the + /// construtor aids migration but is ultimately a thing that should be + /// deprecated and brought internal to the pool, so this is our bridge. + RayletClientFactoryFn defaultClientFactory(rpc::ClientCallManager &ccm) const { + return [&](const rpc::Address &addr) { + auto nm_client = NodeManagerWorkerClient::make(addr.ip_address(), addr.port(), ccm); + std::shared_ptr raylet_client = + std::make_shared(nm_client); + return raylet_client; + }; + }; + + absl::Mutex mu_; + + /// This factory function does the connection to NodeManagerWorkerClient, and is + /// provided by the constructor (either the default implementation, above, or a + /// provided one) + RayletClientFactoryFn client_factory_; + + /// A pool of open connections by host:port. Clients can reuse the connection + /// objects in this pool by requesting them + absl::flat_hash_map> client_map_ + GUARDED_BY(mu_); +}; + +} // namespace rpc +} // namespace ray diff --git a/src/ray/rpc/worker/core_worker_client_pool.cc b/src/ray/rpc/worker/core_worker_client_pool.cc index 794987dfc..8cb9d41b0 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.cc +++ b/src/ray/rpc/worker/core_worker_client_pool.cc @@ -25,7 +25,8 @@ shared_ptr CoreWorkerClientPool::GetOrConnect( auto connection = client_factory_(addr_proto); client_map_[id] = connection; - RAY_LOG(INFO) << "Connected to " << addr_proto.ip_address() << ":" << addr_proto.port(); + RAY_LOG(DEBUG) << "Connected to " << addr_proto.ip_address() << ":" + << addr_proto.port(); return connection; } diff --git a/src/ray/stats/metric.cc b/src/ray/stats/metric.cc index 2cfe236b9..4a475a338 100644 --- a/src/ray/stats/metric.cc +++ b/src/ray/stats/metric.cc @@ -16,6 +16,7 @@ #include "opencensus/stats/internal/aggregation_window.h" #include "opencensus/stats/internal/set_aggregation_window.h" +#include "opencensus/stats/measure_registry.h" namespace ray { @@ -78,14 +79,24 @@ bool StatsConfig::IsInitialized() const { return is_initialized_; } /// /// Metric /// +using MeasureDouble = opencensus::stats::Measure; void Metric::Record(double value, const TagsType &tags) { if (StatsConfig::instance().IsStatsDisabled()) { return; } if (measure_ == nullptr) { - measure_.reset(new opencensus::stats::Measure( - opencensus::stats::Measure::Register(name_, description_, unit_))); + // Measure could be registered before, so we try to get it first. + MeasureDouble registered_measure = + opencensus::stats::MeasureRegistry::GetMeasureDoubleByName(name_); + + if (registered_measure.IsValid()) { + measure_.reset(new MeasureDouble(registered_measure)); + } else { + measure_.reset( + new MeasureDouble(MeasureDouble::Register(name_, description_, unit_))); + } + RegisterView(); } diff --git a/src/ray/stats/metric_defs.h b/src/ray/stats/metric_defs.h index 2c715706c..51afd181b 100644 --- a/src/ray/stats/metric_defs.h +++ b/src/ray/stats/metric_defs.h @@ -36,11 +36,11 @@ static Histogram GcsLatency("gcs_latency", /// Raylet Metrics /// static Gauge LocalAvailableResource("local_available_resource", - "The available resources on this node.", "pcs", + "The available resources on this node.", "", {ResourceNameKey}); static Gauge LocalTotalResource("local_total_resource", - "The total resources on this node.", "pcs", + "The total resources on this node.", "", {ResourceNameKey}); static Gauge LiveActors("live_actors", "Number of live actors.", "actors"); @@ -101,11 +101,11 @@ static Count UnintentionalWorkerFailures( "unintentional_worker_failures_total", "Number of worker failures that are not intentional. For example, worker failures " "due to system related errors.", - "worker_failures"); + ""); static Count NodeFailureTotal( "node_failure_total", "Number of node failures that have happened in the cluster.", - "node_failures."); + ""); static Gauge PendingActors("pending_actors", "Number of pending actors in GCS server.", "actors"); diff --git a/src/ray/util/process.cc b/src/ray/util/process.cc index 6a289ed05..0928c4402 100644 --- a/src/ray/util/process.cc +++ b/src/ray/util/process.cc @@ -495,7 +495,7 @@ void Process::Kill() { } #endif if (error) { - RAY_LOG(ERROR) << "Failed to kill process " << pid << " with error " << error + RAY_LOG(DEBUG) << "Failed to kill process " << pid << " with error " << error << ": " << error.message(); } } else { diff --git a/streaming/README.md b/streaming/README.md deleted file mode 100644 index cb8c5d78d..000000000 --- a/streaming/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Ray Streaming - -1. Build streaming java - * build ray - * `bazel build //java:gen_maven_deps` - * `cd java && mvn clean install -Dmaven.test.skip=true && cd ..` - * build streaming - * `bazel build //streaming/java:gen_maven_deps` - * `mvn clean install -Dmaven.test.skip=true` - -2. Build ray python will build ray streaming python. - -3. Run examples - ```bash - # c++ test - cd streaming/ && bazel test ... - sh src/test/run_streaming_queue_test.sh - cd .. - - # python test - pushd python/ray/streaming/ - pushd examples - python simple.py --input-file toy.txt - popd - pushd tests - pytest . - popd - popd - - # java test - cd streaming/java/streaming-runtime - mvn test - ``` \ No newline at end of file diff --git a/streaming/README.rst b/streaming/README.rst new file mode 100644 index 000000000..90e4a676a --- /dev/null +++ b/streaming/README.rst @@ -0,0 +1,232 @@ + +Ray Streaming +============= + +Ray Streaming is a streaming data processing framework built on ray. It will be helpful for you to build jobs dealing with real-time data. + +Key Features +------------ + + +#. + **Cross Language**. Based on Ray's multi-language actor, Ray Streaming can also run in multiple + languages(only Python and Java is supported currently) with high efficiency. You can implement your + operator in different languages and run them in one job. + +#. + **Single Node Failover**. We designed a special failover mechanism that only needs to rollback the + failed node it's own, in most cases, to recover the job. This will be a huge benefit if your job is + sensitive about failure recovery time. In other frameworks like Flink, instead, the entire job should + be restarted once a node has failure. + +Examples +-------- + +Python +^^^^^^ + +.. code-block:: Python + + import ray + from ray.streaming import StreamingContext + + ctx = StreamingContext.Builder() \ + .build() + ctx.read_text_file(__file__) \ + .set_parallelism(1) \ + .flat_map(lambda x: x.split()) \ + .map(lambda x: (x, 1)) \ + .key_by(lambda x: x[0]) \ + .reduce(lambda old_value, new_value: + (old_value[0], old_value[1] + new_value[1])) \ + .filter(lambda x: "ray" not in x) \ + .sink(lambda x: print("result", x)) + ctx.submit("word_count") + +Java +^^^^ + +.. code-block:: Java + + StreamingContext context = StreamingContext.buildContext(); + List text = Collections.singletonList("hello world"); + DataStreamSource.fromCollection(context, text) + .flatMap((FlatMapFunction) (value, collector) -> { + String[] records = value.split(" "); + for (String record : records) { + collector.collect(new WordAndCount(record, 1)); + } + }) + .filter(pair -> !pair.word.contains("world")) + .keyBy(pair -> pair.word) + .reduce((oldValue, newValue) -> + new WordAndCount(oldValue.word, oldValue.count + newValue.count)) + .sink(result -> System.out.println("sink result=" + result)); + context.execute("testWordCount"); + +Use Java Operators in Python +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: Python + + import ray + from ray.streaming import StreamingContext + + ctx = StreamingContext.Builder().build() + ctx.from_values("a", "b", "c") \ + .as_java_stream() \ + .map("io.ray.streaming.runtime.demo.HybridStreamTest$Mapper1") \ + .filter("io.ray.streaming.runtime.demo.HybridStreamTest$Filter1") \ + .as_python_stream() \ + .sink(lambda x: print("result", x)) + ctx.submit("HybridStreamTest") + +Use Python Operators in Java +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: Java + + StreamingContext context = StreamingContext.buildContext(); + DataStreamSource streamSource = + DataStreamSource.fromCollection(context, Arrays.asList("a", "b", "c")); + streamSource + .map(x -> x + x) + .asPythonStream() + .map("ray.streaming.tests.test_hybrid_stream", "map_func1") + .filter("ray.streaming.tests.test_hybrid_stream", "filter_func1") + .asJavaStream() + .sink(value -> System.out.println("HybridStream sink=" + value)); + context.execute("HybridStreamTestJob"); + +Installation +------------ + +Python +^^^^^^ + +Ray Streaming is packaged together with Ray, install Ray with: ``pip install ray``\ , +this wheel contains all dependencies your need to run Python streaming, including Java operators supporting. + +Java +^^^^ + +Import Ray Streaming using maven: + +.. code-block:: xml + + + ray-api + io.ray + 1.0.1 + + + ray-runtime + io.ray + 1.0.1 + + + streaming-api + io.ray + 1.0.1 + + + streaming-runtime + io.ray + 1.0.1 + + +Internal Design +--------------- + +Overall Architecture +^^^^^^^^^^^^^^^^^^^^ + + +.. image:: assets/architecture.jpg + :target: assets/architecture.jpg + :alt: architecture + + +Ray Streaming is built on Ray. We use Ray's actor to run everything, and use Ray's direct call for communication. + +There are two main types of actor: job master and job worker. + +When you execute ``context.submit()`` in your driver, we'll first create a job master, then job master will create all job workers needed to run your operator. Then job master will be responsible to coordinate all workers, including checkpoint, failover, etc. + +Check `Ray Streaming Proposal `_ +to get more detailed information about the overall design. + +Fault Tolerance Mechanism +^^^^^^^^^^^^^^^^^^^^^^^^^ + +As mentioned above, different from other frameworks, We designed a special failover mechanism that only needs to rollback the failed node it's own, in most cases, to recover the job. The main idea to achieve this feature is saving messages for each node, and replay them from upstream when node has failure. + +Check `Fault Tolerance Proposal `_ +for more detailed information about our fault tolerance mechanism. + +Development Guides +------------------ + + +#. + Build streaming java + + + * build ray + + * ``bazel build //java:gen_maven_deps`` + * ``cd java && mvn clean install -Dmaven.test.skip=true && cd ..`` + + * build streaming + + * ``bazel build //streaming/java:gen_maven_deps`` + * ``mvn clean install -Dmaven.test.skip=true`` + +#. + Build ray python will build ray streaming python. + +#. + Run examples + + .. code-block:: bash + + # c++ test + cd streaming/ && bazel test ... + sh src/test/run_streaming_queue_test.sh + cd .. + + # python test + pushd python/ray/streaming/ + pushd examples + python simple.py --input-file toy.txt + popd + pushd tests + pytest . + popd + popd + + # java test + cd streaming/java/streaming-runtime + mvn test + + +More Information +---------------- + + +* `Ray Streaming implementation plan `_ +* `Fault Tolerance Proposal `_ +* `Data Transfer Proposal `_ +* `Ray Streaming Proposal `_ +* `Open Source Plan `_ + +Getting Involved +---------------- + +- `Community Slack`_: Join our Slack workspace. +- `GitHub Discussions`_: For discussions about development, questions about usage, and feature requests. +- `GitHub Issues`_: For reporting bugs. + +.. _`GitHub Discussions`: https://github.com/ray-project/ray/discussions +.. _`GitHub Issues`: https://github.com/ray-project/ray/issues +.. _`Community Slack`: https://forms.gle/9TSdDYUgxYs8SA9e8 diff --git a/streaming/assets/architecture.jpg b/streaming/assets/architecture.jpg new file mode 100644 index 000000000..9aab86b9a Binary files /dev/null and b/streaming/assets/architecture.jpg differ diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/ClusterStarter.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/ClusterStarter.java index 200ff2e8f..c303b4994 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/ClusterStarter.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/ClusterStarter.java @@ -1,129 +1,28 @@ package io.ray.streaming.api.context; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.gson.Gson; import io.ray.api.Ray; -import io.ray.runtime.config.RayConfig; -import io.ray.runtime.util.NetworkUtil; -import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class ClusterStarter { private static final Logger LOG = LoggerFactory.getLogger(ClusterStarter.class); - private static final String PLASMA_STORE_SOCKET_NAME = "/tmp/ray/plasma_store_socket"; - private static final String RAYLET_SOCKET_NAME = "/tmp/ray/raylet_socket"; - static synchronized void startCluster(boolean isCrossLanguage, boolean isLocal) { + static synchronized void startCluster(boolean isLocal) { Preconditions.checkArgument(!Ray.isInitialized()); - RayConfig.reset(); if (!isLocal) { - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); System.setProperty("ray.run-mode", "CLUSTER"); } else { - System.clearProperty("ray.raylet.config.num_workers_per_process_java"); System.setProperty("ray.run-mode", "SINGLE_PROCESS"); } - if (!isCrossLanguage) { - Ray.init(); - return; - } - - // Delete existing socket files. - for (String socket : ImmutableList.of(RAYLET_SOCKET_NAME, PLASMA_STORE_SOCKET_NAME)) { - File file = new File(socket); - if (file.exists()) { - LOG.info("Delete existing socket file {}", file); - file.delete(); - } - } - - String nodeManagerPort = String.valueOf(NetworkUtil.getUnusedPort()); - - // jars in the `ray` wheel doesn't contains test classes, so we add test classes explicitly. - // Since mvn test classes contains `test` in path and bazel test classes is located at a jar - // with `test` included in the name, we can check classpath `test` to filter out test classes. - String classpath = Stream.of(System.getProperty("java.class.path").split(":")) - .filter(s -> !s.contains(" ") && s.contains("test")) - .collect(Collectors.joining(":")); - String workerOptions = new Gson().toJson(ImmutableList.of("-classpath", classpath)); - Map config = new HashMap<>(RayConfig.create().rayletConfigParameters); - config.put("num_workers_per_process_java", "1"); - // Start ray cluster. - List startCommand = ImmutableList.of( - "ray", - "start", - "--head", - "--port=6379", - String.format("--plasma-store-socket-name=%s", PLASMA_STORE_SOCKET_NAME), - String.format("--raylet-socket-name=%s", RAYLET_SOCKET_NAME), - String.format("--node-manager-port=%s", nodeManagerPort), - "--load-code-from-local", - "--java-worker-options=" + workerOptions, - "--system-config=" + new Gson().toJson(config) - ); - if (!executeCommand(startCommand, 10)) { - throw new RuntimeException("Couldn't start ray cluster."); - } - - // Connect to the cluster. - System.setProperty("ray.address", "127.0.0.1:6379"); - System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME); - System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME); - System.setProperty("ray.raylet.node-manager-port", nodeManagerPort); Ray.init(); } - public static synchronized void stopCluster(boolean isCrossLanguage) { + public static synchronized void stopCluster() { // Disconnect to the cluster. Ray.shutdown(); - System.clearProperty("ray.address"); - System.clearProperty("ray.object-store.socket-name"); - System.clearProperty("ray.raylet.socket-name"); - System.clearProperty("ray.raylet.node-manager-port"); - System.clearProperty("ray.raylet.config.num_workers_per_process_java"); System.clearProperty("ray.run-mode"); - - if (isCrossLanguage) { - // Stop ray cluster. - final List stopCommand = ImmutableList.of( - "ray", - "stop" - ); - if (!executeCommand(stopCommand, 10)) { - throw new RuntimeException("Couldn't stop ray cluster"); - } - } - } - - /** - * Execute an external command. - * - * @return Whether the command succeeded. - */ - private static boolean executeCommand(List command, int waitTimeoutSeconds) { - LOG.info("Executing command: {}", String.join(" ", command)); - try { - ProcessBuilder processBuilder = new ProcessBuilder(command) - .redirectOutput(ProcessBuilder.Redirect.INHERIT) - .redirectError(ProcessBuilder.Redirect.INHERIT); - Process process = processBuilder.start(); - boolean exit = process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); - if (!exit) { - process.destroyForcibly(); - } - return process.exitValue() == 0; - } catch (Exception e) { - throw new RuntimeException("Error executing command " + String.join(" ", command), e); - } } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java index d3264e95f..af97167be 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java @@ -65,11 +65,10 @@ public class StreamingContext implements Serializable { if (!Ray.isInitialized()) { if (Config.MEMORY_CHANNEL.equalsIgnoreCase(jobConfig.get(Config.CHANNEL_TYPE))) { - Preconditions.checkArgument(!jobGraph.isCrossLanguageGraph()); - ClusterStarter.startCluster(false, true); + ClusterStarter.startCluster(true); LOG.info("Created local cluster for job {}.", jobName); } else { - ClusterStarter.startCluster(jobGraph.isCrossLanguageGraph(), false); + ClusterStarter.startCluster(false); LOG.info("Created multi process cluster for job {}.", jobName); } Runtime.getRuntime().addShutdownHook(new Thread(StreamingContext.this::stop)); @@ -103,7 +102,7 @@ public class StreamingContext implements Serializable { public void stop() { if (Ray.isInitialized()) { - ClusterStarter.stopCluster(jobGraph.isCrossLanguageGraph()); + ClusterStarter.stopCluster(); } } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java index cbbf76b75..61722f2f1 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java @@ -1,6 +1,5 @@ package io.ray.streaming.jobgraph; -import io.ray.streaming.api.Language; import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -138,14 +137,4 @@ public class JobGraph implements Serializable { } } - public boolean isCrossLanguageGraph() { - Language language = jobVertices.get(0).getLanguage(); - for (JobVertex jobVertex : jobVertices) { - if (jobVertex.getLanguage() != language) { - return true; - } - } - return false; - } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java index 613c8490a..aa7541468 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java @@ -1,6 +1,6 @@ package io.ray.streaming.runtime.transfer; -import io.ray.runtime.RayNativeRuntime; +import io.ray.runtime.util.BinaryFileUtil; import io.ray.runtime.util.JniUtils; /** @@ -10,11 +10,7 @@ import io.ray.runtime.util.JniUtils; public class TransferHandler { static { - try { - Class.forName(RayNativeRuntime.class.getName()); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } + JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true); JniUtils.loadLibrary("streaming_java"); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java index 29aebdc29..6e14d45af 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java @@ -1,6 +1,6 @@ package io.ray.streaming.runtime.util; -import io.ray.runtime.RayNativeRuntime; +import io.ray.runtime.util.BinaryFileUtil; import io.ray.runtime.util.JniUtils; import java.lang.management.ManagementFactory; import java.net.InetAddress; @@ -29,13 +29,7 @@ public class EnvUtil { } public static void loadNativeLibraries() { - // Explicitly load `RayNativeRuntime`, to make sure `core_worker_library_java` - // is loaded before `streaming_java`. - try { - Class.forName(RayNativeRuntime.class.getName()); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } + JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true); JniUtils.loadLibrary("streaming_java"); } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java index 879364e04..c93d19911 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java @@ -58,11 +58,10 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { void beforeMethod() { LOGGER.info("beforeTest"); Ray.shutdown(); - System.setProperty("ray.resources", "CPU:4,RES-A:4"); - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); + System.setProperty("ray.head-args.0", "--num-cpus=4"); + System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}"); System.setProperty("ray.run-mode", "CLUSTER"); System.setProperty("ray.redirect-output", "true"); - RayConfig.reset(); Ray.init(); } @@ -71,6 +70,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { LOGGER.info("afterTest"); Ray.shutdown(); System.clearProperty("ray.run-mode"); + System.clearProperty("ray.head-args.0"); + System.clearProperty("ray.head-args.1"); } @Test(timeOut = 300000) @@ -78,8 +79,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { LOGGER.info("StreamingQueueTest.testReaderWriter run-mode: {}", System.getProperty("ray.run-mode")); Ray.shutdown(); - System.setProperty("ray.resources", "CPU:4,RES-A:4"); - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); + System.setProperty("ray.head-args.0", "--num-cpus=4"); + System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}"); System.setProperty("ray.run-mode", "CLUSTER"); System.setProperty("ray.redirect-output", "true"); @@ -134,8 +135,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { @Test(timeOut = 60000) public void testWordCount() { Ray.shutdown(); - System.setProperty("ray.resources", "CPU:4,RES-A:4"); - System.setProperty("ray.raylet.config.num_workers_per_process_java", "1"); + System.setProperty("ray.head-args.0", "--num-cpus=4"); + System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}"); System.setProperty("ray.run-mode", "CLUSTER"); System.setProperty("ray.redirect-output", "true"); diff --git a/streaming/python/examples/wordcount.py b/streaming/python/examples/wordcount.py index 2f62b19da..d10782b52 100644 --- a/streaming/python/examples/wordcount.py +++ b/streaming/python/examples/wordcount.py @@ -1,5 +1,6 @@ import argparse import logging +import sys import time import ray @@ -65,7 +66,7 @@ if __name__ == "__main__": args = parser.parse_args() titles_file = str(args.titles_file) - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder() \ .option(Config.CHANNEL_TYPE, Config.NATIVE_CHANNEL) \ diff --git a/streaming/python/tests/test_failover.py b/streaming/python/tests/test_failover.py index adab217e0..264a40099 100644 --- a/streaming/python/tests/test_failover.py +++ b/streaming/python/tests/test_failover.py @@ -1,4 +1,5 @@ import subprocess +import sys import time from typing import List @@ -8,7 +9,8 @@ from ray.streaming import StreamingContext def test_word_count(): try: - ray.init(_load_code_from_local=True) + ray.init( + job_config=ray.job_config.JobConfig(code_search_path=sys.path)) # time.sleep(10) # for gdb to attach ctx = StreamingContext.Builder() \ .option("streaming.context-backend.type", "local_file") \ diff --git a/streaming/python/tests/test_hybrid_stream.py b/streaming/python/tests/test_hybrid_stream.py index e257f0d9f..1a18b3fb1 100644 --- a/streaming/python/tests/test_hybrid_stream.py +++ b/streaming/python/tests/test_hybrid_stream.py @@ -1,6 +1,7 @@ import json import os import subprocess +import sys import ray from ray.streaming import StreamingContext @@ -34,9 +35,8 @@ def test_hybrid_stream(): print("java_worker_options", java_worker_options) assert not ray.is_initialized() ray.init( - _load_code_from_local=True, - _java_worker_options=java_worker_options, - _system_config={"num_workers_per_process_java": 1}) + job_config=ray.job_config.JobConfig(code_search_path=sys.path), + _java_worker_options=java_worker_options) sink_file = "/tmp/ray_streaming_test_hybrid_stream.txt" if os.path.exists(sink_file): diff --git a/streaming/python/tests/test_stream.py b/streaming/python/tests/test_stream.py index f99033d19..27febb822 100644 --- a/streaming/python/tests/test_stream.py +++ b/streaming/python/tests/test_stream.py @@ -1,9 +1,11 @@ +import sys + import ray from ray.streaming import StreamingContext def test_data_stream(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder().build() stream = ctx.from_values(1, 2, 3) java_stream = stream.as_java_stream() @@ -17,7 +19,7 @@ def test_data_stream(): def test_key_data_stream(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder().build() key_stream = ctx.from_values( "a", "b", "c").map(lambda x: (x, 1)).key_by(lambda x: x[0]) @@ -32,7 +34,7 @@ def test_key_data_stream(): def test_stream_config(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder().build() stream = ctx.from_values(1, 2, 3) stream.with_config("k1", "v1") diff --git a/streaming/python/tests/test_union_stream.py b/streaming/python/tests/test_union_stream.py index 0c655b1d0..bab75e624 100644 --- a/streaming/python/tests/test_union_stream.py +++ b/streaming/python/tests/test_union_stream.py @@ -1,11 +1,12 @@ import os +import sys import ray from ray.streaming import StreamingContext def test_union_stream(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder() \ .option("streaming.metrics.reporters", "") \ .build() diff --git a/streaming/python/tests/test_word_count.py b/streaming/python/tests/test_word_count.py index 13b09912c..8275ac6f6 100644 --- a/streaming/python/tests/test_word_count.py +++ b/streaming/python/tests/test_word_count.py @@ -1,11 +1,12 @@ import os +import sys import ray from ray.streaming import StreamingContext from ray.test_utils import wait_for_condition def test_word_count(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder() \ .build() ctx.read_text_file(__file__) \ @@ -24,7 +25,7 @@ def test_word_count(): def test_simple_word_count(): - ray.init(_load_code_from_local=True) + ray.init(job_config=ray.job_config.JobConfig(code_search_path=sys.path)) ctx = StreamingContext.Builder() \ .build() sink_file = "/tmp/ray_streaming_test_simple_word_count.txt" diff --git a/streaming/src/test/mock_actor.cc b/streaming/src/test/mock_actor.cc index 28911758e..09e255995 100644 --- a/streaming/src/test/mock_actor.cc +++ b/streaming/src/test/mock_actor.cc @@ -499,8 +499,8 @@ class StreamingWorker { options.node_ip_address = "127.0.0.1"; options.node_manager_port = node_manager_port; options.raylet_ip_address = "127.0.0.1"; - options.task_execution_callback = - std::bind(&StreamingWorker::ExecuteTask, this, _1, _2, _3, _4, _5, _6, _7, _8); + options.task_execution_callback = std::bind(&StreamingWorker::ExecuteTask, this, _1, + _2, _3, _4, _5, _6, _7, _8, _9); options.ref_counting_enabled = true; options.num_workers = 1; options.metrics_agent_port = -1; @@ -520,6 +520,7 @@ class StreamingWorker { const std::vector> &args, const std::vector &arg_reference_ids, const std::vector &return_ids, + const std::string &debugger_breakpoint, std::vector> *results) { // Only one arg param used in streaming. STREAMING_CHECK(args.size() >= 1) << "args.size() = " << args.size();