diff --git a/.clang-format b/.clang-format index c5ab0983b..5c0f059e1 100644 --- a/.clang-format +++ b/.clang-format @@ -1,6 +1,5 @@ -BasedOnStyle: Chromium -ColumnLimit: 80 +BasedOnStyle: Google +ColumnLimit: 90 DerivePointerAlignment: false IndentCaseLabels: false PointerAlignment: Right -SpaceAfterCStyleCast: true diff --git a/.gitignore b/.gitignore index abd60923e..f8130b3a2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,22 +4,10 @@ /python/build /python/dist /python/flatbuffers-1.7.1/ -/src/common/thirdparty/redis -/src/thirdparty/arrow /flatbuffers-1.7.1/ -/src/thirdparty/boost/ -/src/thirdparty/boost_1_65_1/ -/src/thirdparty/boost_1_60_0/ -/src/thirdparty/catapult/ -/src/thirdparty/flatbuffers/ -/src/thirdparty/parquet-cpp /thirdparty/pkg/ # Files generated by flatc should be ignored -/src/common/format/*.py -/src/common/format/*_generated.h -/src/plasma/format/ -/src/local_scheduler/format/*_generated.h /src/ray/gcs/format/*_generated.h /src/ray/object_manager/format/*_generated.h /src/ray/raylet/format/*_generated.h @@ -54,9 +42,6 @@ python/.eggs *.dylib *.dll -# Cython-generated files -*.c - # Incremental linking files *.ilk diff --git a/.travis.yml b/.travis.yml index 5088b62d9..a2d1b106a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,7 @@ matrix: - sphinx-build -W -b html -d _build/doctrees source _build/html - cd .. # Run Python linting, ignore dict vs {} (C408), others are defaults - - flake8 --exclude=python/ray/core/src/common/flatbuffers_ep-prefix/,python/ray/core/generated/,src/common/format/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 + - flake8 --exclude=python/ray/core/generated/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 - .travis/format.sh --all - os: linux @@ -69,16 +69,9 @@ matrix: script: - cd build - # - bash ../src/common/test/run_valgrind.sh - # - bash ../src/plasma/test/run_valgrind.sh - # - bash ../src/local_scheduler/test/run_valgrind.sh - bash ../src/ray/test/run_object_manager_valgrind.sh - cd .. - # - python ./python/ray/plasma/test/test.py valgrind - # - python ./python/ray/local_scheduler/test/test.py valgrind - # - python ./python/ray/global_scheduler/test/test.py valgrind - # Build Linux wheels. - os: linux dist: trusty @@ -108,75 +101,6 @@ matrix: - PYTHON=3.5 - RAY_USE_NEW_GCS=on - # Test legacy Ray. - - os: linux - dist: trusty - env: PYTHON=3.5 RAY_USE_XRAY=0 - install: - - ./.travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./.travis/install-ray.sh - - ./.travis/install-cython-examples.sh - - - cd build - - bash ../src/common/test/run_tests.sh - - bash ../src/plasma/test/run_tests.sh - - bash ../src/local_scheduler/test/run_tests.sh - - cd .. - - script: - - export PATH="$HOME/miniconda/bin:$PATH" - # The following is needed so cloudpickle can find some of the - # class definitions: The main module of tests that are run - # with pytest have the same name as the test file -- and this - # module is only found if the test directory is in the PYTHONPATH. - - export PYTHONPATH="$PYTHONPATH:./test/" - - - python -m pytest -v python/ray/common/test/test.py - - python -m pytest -v python/ray/common/redis_module/runtest.py - - python -m pytest -v python/ray/plasma/test/test.py - - python -m pytest -v python/ray/local_scheduler/test/test.py - - python -m pytest -v python/ray/global_scheduler/test/test.py - - - python -m pytest -v python/ray/test/test_global_state.py - - python -m pytest -v python/ray/test/test_queue.py - - python -m pytest -v python/ray/test/test_ray_init.py - - python -m pytest -v test/xray_test.py - - - python -m pytest -v test/runtest.py - - python -m pytest -v test/array_test.py - - python -m pytest -v test/actor_test.py - - python -m pytest -v test/autoscaler_test.py - - python -m pytest -v test/tensorflow_test.py - - python -m pytest -v test/failure_test.py - - python -m pytest -v test/microbenchmarks.py - - python -m pytest -v test/stress_tests.py - - pytest test/component_failures_test.py - - python test/multi_node_test.py - - python -m pytest -v test/multi_node_test_2.py - - python -m pytest -v test/recursion_test.py - - pytest test/monitor_test.py - - python -m pytest -v test/cython_test.py - - python -m pytest -v test/credis_test.py - - # ray tune tests - - python python/ray/tune/test/dependency_test.py - - python -m pytest -v python/ray/tune/test/trial_runner_test.py - - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py - - python -m pytest -v python/ray/tune/test/experiment_test.py - - python -m pytest -v python/ray/tune/test/tune_server_test.py - - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py - - python -m pytest -v python/ray/tune/test/automl_searcher_test.py - - # ray rllib tests - - python -m pytest -v python/ray/rllib/test/test_catalog.py - - python -m pytest -v python/ray/rllib/test/test_filters.py - - python -m pytest -v python/ray/rllib/test/test_optimizers.py - - python -m pytest -v python/ray/rllib/test/test_evaluators.py - - # ray temp file tests - - python -m pytest -v test/tempfile_test.py - install: - ./.travis/install-dependencies.sh @@ -207,12 +131,6 @@ script: # module is only found if the test directory is in the PYTHONPATH. - export PYTHONPATH="$PYTHONPATH:./test/" - # - python -m pytest -v python/ray/common/test/test.py - # - python -m pytest -v python/ray/common/redis_module/runtest.py - # - python -m pytest -v python/ray/plasma/test/test.py - # - python -m pytest -v python/ray/local_scheduler/test/test.py - # - python -m pytest -v python/ray/global_scheduler/test/test.py - - python -m pytest -v python/ray/test/test_global_state.py - python -m pytest -v python/ray/test/test_queue.py - python -m pytest -v python/ray/test/test_ray_init.py @@ -227,7 +145,7 @@ script: - python -m pytest -v test/microbenchmarks.py - python -m pytest -v test/stress_tests.py - python -m pytest -v test/component_failures_test.py - - python test/multi_node_test.py + - python -m pytest -v test/multi_node_test.py - python -m pytest -v test/multi_node_test_2.py - python -m pytest -v test/recursion_test.py - python -m pytest -v test/monitor_test.py diff --git a/.travis/format.sh b/.travis/format.sh index ca92d5196..e4c609a85 100755 --- a/.travis/format.sh +++ b/.travis/format.sh @@ -30,7 +30,6 @@ YAPF_EXCLUDES=( '--exclude' 'python/build/*' '--exclude' 'python/ray/pyarrow_files/*' '--exclude' 'python/ray/core/src/ray/gcs/*' - '--exclude' 'python/ray/common/thirdparty/*' ) # Format specified files diff --git a/CMakeLists.txt b/CMakeLists.txt index d02e88a5c..3980de004 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,18 +82,15 @@ include_directories(SYSTEM ${PLASMA_INCLUDE_DIR}) include_directories("${CMAKE_CURRENT_LIST_DIR}/src/") add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/ray/) -add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/) -add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/) -add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/) -add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/) # final target copy_ray add_custom_target(copy_ray ALL) # copy plasma_store_server add_custom_command(TARGET copy_ray POST_BUILD + COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma COMMAND ${CMAKE_COMMAND} -E - copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma) + copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/) if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") # add pyarrow as the dependency @@ -102,12 +99,9 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") # NOTE: The lists below must be kept in sync with ray/python/setup.py. set(ray_file_list - "src/common/thirdparty/redis/src/redis-server" - "src/common/redis_module/libray_redis_module.so" - "src/plasma/plasma_manager" - "src/local_scheduler/local_scheduler" - "src/local_scheduler/liblocal_scheduler_library_python.so" - "src/global_scheduler/global_scheduler" + "src/ray/thirdparty/redis/src/redis-server" + "src/ray/gcs/redis_module/libray_redis_module.so" + "src/ray/raylet/liblocal_scheduler_library_python.so" "src/ray/raylet/raylet_monitor" "src/ray/raylet/raylet") @@ -154,5 +148,6 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") # copy libplasma_java files add_custom_command(TARGET copy_ray POST_BUILD - COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma") + COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma" + COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/") endif() diff --git a/cmake/Modules/ArrowExternalProject.cmake b/cmake/Modules/ArrowExternalProject.cmake index 08d250b81..b57104f1f 100644 --- a/cmake/Modules/ArrowExternalProject.cmake +++ b/cmake/Modules/ArrowExternalProject.cmake @@ -15,10 +15,10 @@ # - PLASMA_SHARED_LIB set(arrow_URL https://github.com/apache/arrow.git) -# The PR for this commit is https://github.com/apache/arrow/pull/2792. We +# The PR for this commit is https://github.com/apache/arrow/pull/2826. We # include the link here to make it easier to find the right commit because # Arrow often rewrites git history and invalidates certain commits. -set(arrow_TAG 2d0d3d0dc51999fbaafb15d8b8362a1ef3de2ef7) +set(arrow_TAG b4f7ed6d6ed5cdb6dd136bac3181a438f35c8ea0) set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install) set(ARROW_HOME ${ARROW_INSTALL_PREFIX}) diff --git a/cmake/Modules/Common.cmake b/cmake/Modules/Common.cmake index cc2a5d5ff..7d33f13e9 100644 --- a/cmake/Modules/Common.cmake +++ b/cmake/Modules/Common.cmake @@ -41,6 +41,3 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") message (WARNING "NOT FIND JNI") endif() endif() - -include_directories(${CMAKE_SOURCE_DIR}/src/common) -include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty) diff --git a/doc/source/actors.rst b/doc/source/actors.rst index c7594592f..0d8b3c942 100644 --- a/doc/source/actors.rst +++ b/doc/source/actors.rst @@ -65,8 +65,7 @@ When ``a1.increment.remote()`` is called, the following events happens. 1. A task is created. 2. The task is assigned directly to the local scheduler responsible for the - actor by the driver's local scheduler. Thus, this scheduling procedure - bypasses the global scheduler. + actor by the driver's local scheduler. 3. An object ID is returned. We can then call ``ray.get`` on the object ID to retrieve the actual value. diff --git a/doc/source/conf.py b/doc/source/conf.py index 27d0c1200..2d212d23b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,44 +18,38 @@ import shlex # These lines added to enable Sphinx to work without installing Ray. import mock -MOCK_MODULES = ["gym", - "gym.spaces", - "scipy", - "scipy.signal", - "tensorflow", - "tensorflow.contrib", - "tensorflow.contrib.layers", - "tensorflow.contrib.slim", - "tensorflow.contrib.rnn", - "tensorflow.core", - "tensorflow.core.util", - "tensorflow.python", - "tensorflow.python.client", - "tensorflow.python.util", - "ray.local_scheduler", - "ray.plasma", - "ray.core", - "ray.core.generated", - "ray.core.generated.DriverTableMessage", - "ray.core.generated.LocalSchedulerInfoMessage", - "ray.core.generated.ResultTableReply", - "ray.core.generated.SubscribeToDBClientTableReply", - "ray.core.generated.SubscribeToNotificationsReply", - "ray.core.generated.TaskInfo", - "ray.core.generated.TaskReply", - "ray.core.generated.TaskExecutionDependencies", - "ray.core.generated.ClientTableData", - "ray.core.generated.GcsTableEntry", - "ray.core.generated.HeartbeatTableData", - "ray.core.generated.DriverTableData", - "ray.core.generated.ErrorTableData", - "ray.core.generated.ProfileTableData", - "ray.core.generated.ObjectTableData", - "ray.core.generated.ray.protocol.Task", - "ray.core.generated.TablePrefix", - "ray.core.generated.TablePubsub",] +MOCK_MODULES = [ + "gym", + "gym.spaces", + "scipy", + "scipy.signal", + "tensorflow", + "tensorflow.contrib", + "tensorflow.contrib.layers", + "tensorflow.contrib.slim", + "tensorflow.contrib.rnn", + "tensorflow.core", + "tensorflow.core.util", + "tensorflow.python", + "tensorflow.python.client", + "tensorflow.python.util", + "ray.raylet", + "ray.plasma", + "ray.core", + "ray.core.generated", + "ray.core.generated.ClientTableData", + "ray.core.generated.GcsTableEntry", + "ray.core.generated.HeartbeatTableData", + "ray.core.generated.DriverTableData", + "ray.core.generated.ErrorTableData", + "ray.core.generated.ProfileTableData", + "ray.core.generated.ObjectTableData", + "ray.core.generated.ray.protocol.Task", + "ray.core.generated.TablePrefix", + "ray.core.generated.TablePubsub", +] for mod_name in MOCK_MODULES: - sys.modules[mod_name] = mock.Mock() + sys.modules[mod_name] = mock.Mock() # ray.rllib.models.action_dist.py and # ray.rllib.models.lstm.py will use tf.VERSION sys.modules["tensorflow"].VERSION = "9.9.9" @@ -89,7 +83,7 @@ from recommonmark.parser import CommonMarkParser source_suffix = ['.rst', '.md'] source_parsers = { - '.md': CommonMarkParser, + '.md': CommonMarkParser, } # The encoding of source files. @@ -259,25 +253,24 @@ htmlhelp_basename = 'Raydoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + #'preamble': '', -# Latex figure (float) alignment -#'figure_align': 'htbp', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'Ray.tex', u'Ray Documentation', - u'The Ray Team', 'manual'), + (master_doc, 'Ray.tex', u'Ray Documentation', u'The Ray Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -300,29 +293,23 @@ latex_documents = [ # If false, no module index is generated. #latex_domain_indices = True - # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'ray', u'Ray Documentation', - [author], 1) -] +man_pages = [(master_doc, 'ray', u'Ray Documentation', [author], 1)] # If true, show URL addresses after external links. #man_show_urls = False - # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'Ray', u'Ray Documentation', - author, 'Ray', 'One line description of project.', - 'Miscellaneous'), + (master_doc, 'Ray', u'Ray Documentation', author, 'Ray', + 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/doc/source/fault-tolerance.rst b/doc/source/fault-tolerance.rst index a4692f904..112927ce3 100644 --- a/doc/source/fault-tolerance.rst +++ b/doc/source/fault-tolerance.rst @@ -47,7 +47,7 @@ Process Failures ~~~~~~~~~~~~~~~~ 1. Ray does not recover from the failure of any of the following processes: - a Redis server, the global scheduler, the monitor process. + a Redis server and the monitor process. 2. If a driver fails, that driver will not be restarted and the job will not complete. diff --git a/doc/source/internals-overview.rst b/doc/source/internals-overview.rst index 69ac1895a..a2516de1d 100644 --- a/doc/source/internals-overview.rst +++ b/doc/source/internals-overview.rst @@ -15,8 +15,8 @@ Running Ray standalone Ray can be used standalone by calling ``ray.init()`` within a script. When the call to ``ray.init()`` happens, all of the relevant processes are started. -These include a local scheduler, a global scheduler, an object store and -manager, a Redis server, and a number of worker processes. +These include a local scheduler, an object store and manager, a Redis server, +and a number of worker processes. When the script exits, these processes will be killed. @@ -112,7 +112,7 @@ When a driver or worker invokes a remote function, a number of things happen. - The task object is then sent to the local scheduler on the same node as the driver or worker. - The local scheduler makes a decision to either schedule the task locally or to - pass the task on to a global scheduler. + pass the task on to another local scheduler. - If all of the task's object dependencies are present in the local object store and there are enough CPU and GPU resources available to execute the diff --git a/doc/source/tempfile.rst b/doc/source/tempfile.rst index 7e489348c..99daf2833 100644 --- a/doc/source/tempfile.rst +++ b/doc/source/tempfile.rst @@ -45,8 +45,6 @@ A typical layout of temporary files could look like this: │   ├── log_monitor.out │   ├── monitor.err │   ├── monitor.out - │   ├── plasma_manager_0.err # array of plasma managers' outputs - │   ├── plasma_manager_0.out │   ├── plasma_store_0.err # array of plasma stores' outputs │   ├── plasma_store_0.out │   ├── raylet_0.err # array of raylets' outputs. Control it with `--no-redirect-worker-output` (in Ray's command line) or `redirect_worker_output` (in ray.init()) diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 0493b6916..81de87a57 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -9,7 +9,7 @@ To use Ray, you need to understand the following: Overview -------- -Ray is a Python-based distributed execution engine. The same code can be run on +Ray is a distributed execution engine. The same code can be run on a single machine to achieve efficient multiprocessing, and it can be used on a cluster for large computations. @@ -21,8 +21,6 @@ When using Ray, several processes are involved. allows workers to efficiently share objects on the same node with minimal copying and deserialization. - One **local scheduler** per node assigns tasks to workers on the same node. -- A **global scheduler** receives tasks from local schedulers and assigns them - to other local schedulers. - A **driver** is the Python process that the user controls. For example, if the user is running a script or using a Python shell, then the driver is the Python process that runs the script or the shell. A driver is similar to a worker in diff --git a/doc/source/using-ray-on-a-cluster.rst b/doc/source/using-ray-on-a-cluster.rst index 29c2585ac..611e47b79 100644 --- a/doc/source/using-ray-on-a-cluster.rst +++ b/doc/source/using-ray-on-a-cluster.rst @@ -51,7 +51,6 @@ Now we've started all of the Ray processes on each node Ray. This includes - An object store on each machine. - A local scheduler on each machine. - Multiple Redis servers (on the head node). -- One global scheduler (on the head node). To run some commands, start up Python on one of the nodes in the cluster, and do the following. diff --git a/doc/source/using-ray-on-a-large-cluster.rst b/doc/source/using-ray-on-a-large-cluster.rst index c3d6d8a8d..b87c8c05f 100644 --- a/doc/source/using-ray-on-a-large-cluster.rst +++ b/doc/source/using-ray-on-a-large-cluster.rst @@ -154,7 +154,6 @@ Now you have started all of the Ray processes on each node. These include: - An object store on each machine. - A local scheduler on each machine. - Multiple Redis servers (on the head node). -- One global scheduler (on the head node). To confirm that the Ray cluster setup is working, start up Python on one of the nodes in the cluster and enter the following commands to connect to the Ray diff --git a/java/checkstyle-suppressions.xml b/java/checkstyle-suppressions.xml index 619c24e14..042233225 100644 --- a/java/checkstyle-suppressions.xml +++ b/java/checkstyle-suppressions.xml @@ -10,5 +10,5 @@ - + diff --git a/java/prepare.sh b/java/prepare.sh index 807301a74..9554e500a 100755 --- a/java/prepare.sh +++ b/java/prepare.sh @@ -42,15 +42,15 @@ fi # echo "ray_dir = $ray_dir" declare -a nativeBinaries=( - "./src/common/thirdparty/redis/src/redis-server" + "./src/ray/thirdparty/redis/src/redis-server" "./src/plasma/plasma_store_server" "./src/ray/raylet/raylet" "./src/ray/raylet/raylet_monitor" ) declare -a nativeLibraries=( - "./src/common/redis_module/libray_redis_module.so" - "./src/local_scheduler/liblocal_scheduler_library_java.*" + "./src/ray/gcs/redis_module/libray_redis_module.so" + "./src/ray/raylet/liblocal_scheduler_library_java.*" "./src/plasma/libplasma_java.*" "./src/ray/raylet/*lib.a" ) diff --git a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java index e07b9e89e..d4d90f24e 100644 --- a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java @@ -165,12 +165,12 @@ public class RayConfig { // library path this.libraryPath = new ImmutableList.Builder().add( rayHome + "/build/src/plasma", - rayHome + "/build/src/local_scheduler" + rayHome + "/build/src/ray/raylet" ).addAll(customLibraryPath).build(); redisServerExecutablePath = rayHome + - "/build/src/common/thirdparty/redis/src/redis-server"; - redisModulePath = rayHome + "/build/src/common/redis_module/libray_redis_module.so"; + "/build/src/ray/thirdparty/redis/src/redis-server"; + redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so"; plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server"; rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet"; diff --git a/java/runtime/src/main/java/org/ray/runtime/generated/TaskLanguage.java b/java/runtime/src/main/java/org/ray/runtime/generated/Language.java similarity index 51% rename from java/runtime/src/main/java/org/ray/runtime/generated/TaskLanguage.java rename to java/runtime/src/main/java/org/ray/runtime/generated/Language.java index e5e53614a..34604374d 100644 --- a/java/runtime/src/main/java/org/ray/runtime/generated/TaskLanguage.java +++ b/java/runtime/src/main/java/org/ray/runtime/generated/Language.java @@ -2,13 +2,13 @@ package org.ray.runtime.generated; -public final class TaskLanguage { - private TaskLanguage() { } +public final class Language { + private Language() { } public static final int PYTHON = 0; - public static final int JAVA = 1; + public static final int CPP = 1; + public static final int JAVA = 2; - public static final String[] names = { "PYTHON", "JAVA", }; + public static final String[] names = { "PYTHON", "CPP", "JAVA", }; public static String name(int e) { return names[e]; } } - diff --git a/java/runtime/src/main/java/org/ray/runtime/raylet/RayletClientImpl.java b/java/runtime/src/main/java/org/ray/runtime/raylet/RayletClientImpl.java index b84fe22db..28f0cd97c 100644 --- a/java/runtime/src/main/java/org/ray/runtime/raylet/RayletClientImpl.java +++ b/java/runtime/src/main/java/org/ray/runtime/raylet/RayletClientImpl.java @@ -13,9 +13,9 @@ import org.ray.api.WaitResult; import org.ray.api.id.UniqueId; import org.ray.runtime.functionmanager.FunctionDescriptor; import org.ray.runtime.generated.Arg; +import org.ray.runtime.generated.Language; import org.ray.runtime.generated.ResourcePair; import org.ray.runtime.generated.TaskInfo; -import org.ray.runtime.generated.TaskLanguage; import org.ray.runtime.task.FunctionArg; import org.ray.runtime.task.TaskSpec; import org.ray.runtime.util.UniqueIdUtil; @@ -229,7 +229,7 @@ public class RayletClientImpl implements RayletClient { actorIdOffset, actorHandleIdOffset, actorCounter, false, functionIdOffset, argsOffset, returnsOffset, requiredResourcesOffset, - requiredPlacementResourcesOffset, TaskLanguage.JAVA, + requiredPlacementResourcesOffset, Language.JAVA, functionDescriptorOffset); fbb.finish(root); ByteBuffer buffer = fbb.dataBuffer(); @@ -256,8 +256,8 @@ public class RayletClientImpl implements RayletClient { /// 1) pushd $Dir/java/runtime/target/classes /// 2) javah -classpath .:$Dir/java/api/target/classes org.ray.runtime.raylet.RayletClientImpl /// 3) clang-format -i org_ray_runtime_raylet_RayletClientImpl.h - /// 4) cp org_ray_runtime_raylet_RayletClientImpl.h $Dir/src/local_scheduler/lib/java/ - /// 5) vim $Dir/src/local_scheduler/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc + /// 4) cp org_ray_runtime_raylet_RayletClientImpl.h $Dir/src/ray/raylet/lib/java/ + /// 5) vim $Dir/src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc /// 6) popd private static native long nativeInit(String localSchedulerSocket, byte[] workerId, diff --git a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java index ac7e01124..71e3d0dff 100644 --- a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java +++ b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java @@ -23,7 +23,7 @@ public class RayConfigTest { Assert.assertEquals(System.getProperty("user.dir"), rayConfig.rayHome); Assert.assertEquals(System.getProperty("user.dir") + - "/build/src/common/thirdparty/redis/src/redis-server", rayConfig.redisServerExecutablePath); + "/build/src/ray/thirdparty/redis/src/redis-server", rayConfig.redisServerExecutablePath); Assert.assertEquals("path/to/ray/driver/resource/path", rayConfig.driverResourcePath); diff --git a/java/tutorial/pom.xml b/java/tutorial/pom.xml index 198f6f0a3..48a03dc1c 100644 --- a/java/tutorial/pom.xml +++ b/java/tutorial/pom.xml @@ -40,7 +40,7 @@ ${basedir}/../ray.config.ini -ea - -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/local_scheduler + -Djava.library.path=${basedir}/../../build/src/plasma:${basedir}/../../build/src/ray/raylet -noverify -DlogOutput=console diff --git a/python/ray/__init__.py b/python/ray/__init__.py index b97af4b58..95255bba1 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -46,7 +46,7 @@ except ImportError as e: e.args += (helpful_message, ) raise -from ray.local_scheduler import ObjectID, _config # noqa: E402 +from ray.raylet import ObjectID, _config # noqa: E402 from ray.profiling import profile # noqa: E402 from ray.worker import (error_info, init, connect, disconnect, get, put, wait, remote, get_gpu_ids, get_resource_ids, get_webui_url, diff --git a/python/ray/actor.py b/python/ray/actor.py index d1f034cc6..86b87e7d5 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -9,7 +9,7 @@ import traceback import ray.cloudpickle as pickle from ray.function_manager import FunctionActorManager -import ray.local_scheduler +import ray.raylet import ray.ray_constants as ray_constants import ray.signature as signature import ray.worker diff --git a/python/ray/common/redis_module/.gitkeep b/python/ray/common/redis_module/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/ray/common/redis_module/runtest.py b/python/ray/common/redis_module/runtest.py deleted file mode 100644 index 7a7d25c6b..000000000 --- a/python/ray/common/redis_module/runtest.py +++ /dev/null @@ -1,451 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import redis -import sys -import time -import unittest - -import ray.gcs_utils -import ray.services - - -def integerToAsciiHex(num, numbytes): - retstr = b"" - # Support 32 and 64 bit architecture. - assert (numbytes == 4 or numbytes == 8) - for i in range(numbytes): - curbyte = num & 0xff - if sys.version_info >= (3, 0): - retstr += bytes([curbyte]) - else: - retstr += chr(curbyte) - num = num >> 8 - - return retstr - - -def get_next_message(pubsub_client, timeout_seconds=10): - """Block until the next message is available on the pubsub channel.""" - start_time = time.time() - while True: - message = pubsub_client.get_message() - if message is not None: - return message - time.sleep(0.1) - if time.time() - start_time > timeout_seconds: - raise Exception("Timed out while waiting for next message.") - - -class TestGlobalStateStore(unittest.TestCase): - def setUp(self): - unused_primary_redis_addr, redis_shards = ray.services.start_redis( - "localhost", use_credis="RAY_USE_NEW_GCS" in os.environ) - self.redis = redis.StrictRedis( - host="localhost", port=redis_shards[0].split(":")[-1], db=0) - - def tearDown(self): - ray.services.cleanup() - - def testInvalidObjectTableAdd(self): - # Check that Redis returns an error when RAY.OBJECT_TABLE_ADD is called - # with the wrong arguments. - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "hello") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id2", - "one", "hash2", "manager_id1") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id2", 1, - "hash2", "manager_id1", - "extra argument") - # Check that Redis returns an error when RAY.OBJECT_TABLE_ADD adds an - # object ID that is already present with a different hash. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id1") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1"}) - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash2", "manager_id2") - # Check that the second manager was added, even though the hash was - # mismatched. - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - # Check that it is fine if we add the same object ID multiple times - # with the most recent hash. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash2", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash2", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash2", "manager_id2") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 2, - "hash2", "manager_id2") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - - def testObjectTableAddAndLookup(self): - # Try calling RAY.OBJECT_TABLE_LOOKUP with an object ID that has not - # been added yet. - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(response, None) - # Add some managers and try again. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id2") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - # Add a manager that already exists again and try again. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id2") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - # Check that we properly handle NULL characters. In the past, NULL - # characters were handled improperly causing a "hash mismatch" error if - # two object IDs that agreed up to the NULL character were inserted - # with different hashes. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "\x00object_id3", 1, - "hash1", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "\x00object_id4", 1, - "hash2", "manager_id1") - # Check that NULL characters in the hash are handled properly. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id3", 1, - "\x00hash1", "manager_id1") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id3", 1, - "\x00hash2", "manager_id1") - - def testObjectTableAddAndRemove(self): - # Try removing a manager from an object ID that has not been added yet. - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id1") - # Try calling RAY.OBJECT_TABLE_LOOKUP with an object ID that has not - # been added yet. - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(response, None) - # Add some managers and try again. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id2") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - # Remove a manager that doesn't exist, and make sure we still have the - # same set. - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id3") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id1", b"manager_id2"}) - # Remove a manager that does exist. Make sure it gets removed the first - # time and does nothing the second time. - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id1") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id2"}) - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id1") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), {b"manager_id2"}) - # Remove the last manager, and make sure we have an empty set. - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id2") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), set()) - # Remove a manager from an empty set, and make sure we now have an - # empty set. - self.redis.execute_command("RAY.OBJECT_TABLE_REMOVE", "object_id1", - "manager_id3") - response = self.redis.execute_command("RAY.OBJECT_TABLE_LOOKUP", - "object_id1") - self.assertEqual(set(response), set()) - - def testObjectTableSubscribeToNotifications(self): - # Define a helper method for checking the contents of object - # notifications. - def check_object_notification(notification_message, object_id, - object_size, manager_ids): - notification_object = (ray.gcs_utils.SubscribeToNotificationsReply. - GetRootAsSubscribeToNotificationsReply( - notification_message, 0)) - self.assertEqual(notification_object.ObjectId(), object_id) - self.assertEqual(notification_object.ObjectSize(), object_size) - self.assertEqual(notification_object.ManagerIdsLength(), - len(manager_ids)) - for i in range(len(manager_ids)): - self.assertEqual( - notification_object.ManagerIds(i), manager_ids[i]) - - data_size = 0xf1f0 - p = self.redis.pubsub() - # Subscribe to an object ID. - p.psubscribe("{}manager_id1".format( - ray.gcs_utils.OBJECT_CHANNEL_PREFIX)) - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", - data_size, "hash1", "manager_id2") - # Receive the acknowledgement message. - self.assertEqual(get_next_message(p)["data"], 1) - # Request a notification and receive the data. - self.redis.execute_command("RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS", - "manager_id1", "object_id1") - # Verify that the notification is correct. - check_object_notification( - get_next_message(p)["data"], b"object_id1", data_size, - [b"manager_id2"]) - - # Request a notification for an object that isn't there. Then add the - # object and receive the data. Only the first call to - # RAY.OBJECT_TABLE_ADD should trigger notifications. - self.redis.execute_command("RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS", - "manager_id1", "object_id2", "object_id3") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id3", - data_size, "hash1", "manager_id1") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id3", - data_size, "hash1", "manager_id2") - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id3", - data_size, "hash1", "manager_id3") - # Verify that the notification is correct. - check_object_notification( - get_next_message(p)["data"], b"object_id3", data_size, - [b"manager_id1"]) - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id2", - data_size, "hash1", "manager_id3") - # Verify that the notification is correct. - check_object_notification( - get_next_message(p)["data"], b"object_id2", data_size, - [b"manager_id3"]) - # Request notifications for object_id3 again. - self.redis.execute_command("RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS", - "manager_id1", "object_id3") - # Verify that the notification is correct. - check_object_notification( - get_next_message(p)["data"], b"object_id3", data_size, - [b"manager_id1", b"manager_id2", b"manager_id3"]) - - def testResultTableAddAndLookup(self): - def check_result_table_entry(message, task_id, is_put): - result_table_reply = ( - ray.gcs_utils.ResultTableReply.GetRootAsResultTableReply( - message, 0)) - self.assertEqual(result_table_reply.TaskId(), task_id) - self.assertEqual(result_table_reply.IsPut(), is_put) - - # Try looking up something in the result table before anything is - # added. - response = self.redis.execute_command("RAY.RESULT_TABLE_LOOKUP", - "object_id1") - self.assertIsNone(response) - # Adding the object to the object table should have no effect. - self.redis.execute_command("RAY.OBJECT_TABLE_ADD", "object_id1", 1, - "hash1", "manager_id1") - response = self.redis.execute_command("RAY.RESULT_TABLE_LOOKUP", - "object_id1") - self.assertIsNone(response) - # Add the result to the result table. The lookup now returns the task - # ID. - task_id = b"task_id1" - self.redis.execute_command("RAY.RESULT_TABLE_ADD", "object_id1", - task_id, 0) - response = self.redis.execute_command("RAY.RESULT_TABLE_LOOKUP", - "object_id1") - check_result_table_entry(response, task_id, False) - # Doing it again should still work. - response = self.redis.execute_command("RAY.RESULT_TABLE_LOOKUP", - "object_id1") - check_result_table_entry(response, task_id, False) - # Try another result table lookup. This should succeed. - task_id = b"task_id2" - self.redis.execute_command("RAY.RESULT_TABLE_ADD", "object_id2", - task_id, 1) - response = self.redis.execute_command("RAY.RESULT_TABLE_LOOKUP", - "object_id2") - check_result_table_entry(response, task_id, True) - - def testInvalidTaskTableAdd(self): - # Check that Redis returns an error when RAY.TASK_TABLE_ADD is called - # with the wrong arguments. - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.TASK_TABLE_ADD") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.TASK_TABLE_ADD", "hello") - with self.assertRaises(redis.ResponseError): - self.redis.execute_command("RAY.TASK_TABLE_ADD", "task_id", 3, - "node_id") - with self.assertRaises(redis.ResponseError): - # Non-integer scheduling states should not be added. - self.redis.execute_command("RAY.TASK_TABLE_ADD", "task_id", - "invalid_state", "node_id", "task_spec") - with self.assertRaises(redis.ResponseError): - # Should not be able to update a non-existent task. - self.redis.execute_command("RAY.TASK_TABLE_UPDATE", "task_id", 10, - "node_id", b"") - - def testTaskTableAddAndLookup(self): - TASK_STATUS_WAITING = 1 - TASK_STATUS_SCHEDULED = 2 - TASK_STATUS_QUEUED = 4 - - # make sure somebody will get a notification (checked in the redis - # module) - p = self.redis.pubsub() - p.psubscribe("{prefix}*:*".format(prefix=ray.gcs_utils.TASK_PREFIX)) - - def check_task_reply(message, task_args, updated=False): - (task_status, local_scheduler_id, execution_dependencies_string, - spillback_count, task_spec) = task_args - task_reply_object = ray.gcs_utils.TaskReply.GetRootAsTaskReply( - message, 0) - self.assertEqual(task_reply_object.State(), task_status) - self.assertEqual(task_reply_object.LocalSchedulerId(), - local_scheduler_id) - self.assertEqual(task_reply_object.SpillbackCount(), - spillback_count) - self.assertEqual(task_reply_object.TaskSpec(), task_spec) - self.assertEqual(task_reply_object.Updated(), updated) - - # Check that task table adds, updates, and lookups work correctly. - task_args = [TASK_STATUS_WAITING, b"node_id", b"", 0, b"task_spec"] - response = self.redis.execute_command("RAY.TASK_TABLE_ADD", "task_id", - *task_args) - response = self.redis.execute_command("RAY.TASK_TABLE_GET", "task_id") - check_task_reply(response, task_args) - - task_args[0] = TASK_STATUS_SCHEDULED - self.redis.execute_command("RAY.TASK_TABLE_UPDATE", "task_id", - *task_args[:4]) - response = self.redis.execute_command("RAY.TASK_TABLE_GET", "task_id") - check_task_reply(response, task_args) - - # If the current value, test value, and set value are all the same, the - # update happens, and the response is still the same task. - task_args = [task_args[0]] + task_args - response = self.redis.execute_command("RAY.TASK_TABLE_TEST_AND_UPDATE", - "task_id", *task_args[:3]) - check_task_reply(response, task_args[1:], updated=True) - # Check that the task entry is still the same. - get_response = self.redis.execute_command("RAY.TASK_TABLE_GET", - "task_id") - check_task_reply(get_response, task_args[1:]) - - # If the current value is the same as the test value, and the set value - # is different, the update happens, and the response is the entire - # task. - task_args[1] = TASK_STATUS_QUEUED - response = self.redis.execute_command("RAY.TASK_TABLE_TEST_AND_UPDATE", - "task_id", *task_args[:3]) - check_task_reply(response, task_args[1:], updated=True) - # Check that the update happened. - get_response = self.redis.execute_command("RAY.TASK_TABLE_GET", - "task_id") - check_task_reply(get_response, task_args[1:]) - - # If the current value is no longer the same as the test value, the - # response is the same task as before the test-and-set. - new_task_args = task_args[:] - new_task_args[1] = TASK_STATUS_WAITING - response = self.redis.execute_command("RAY.TASK_TABLE_TEST_AND_UPDATE", - "task_id", *new_task_args[:3]) - check_task_reply(response, task_args[1:], updated=False) - # Check that the update did not happen. - get_response2 = self.redis.execute_command("RAY.TASK_TABLE_GET", - "task_id") - self.assertEqual(get_response2, get_response) - - # If the test value is a bitmask that matches the current value, the - # update happens. - task_args = new_task_args - task_args[0] = TASK_STATUS_SCHEDULED | TASK_STATUS_QUEUED - response = self.redis.execute_command("RAY.TASK_TABLE_TEST_AND_UPDATE", - "task_id", *task_args[:3]) - check_task_reply(response, task_args[1:], updated=True) - - # If the test value is a bitmask that does not match the current value, - # the update does not happen, and the response is the same task as - # before the test-and-set. - new_task_args = task_args[:] - new_task_args[0] = TASK_STATUS_SCHEDULED - old_response = response - response = self.redis.execute_command("RAY.TASK_TABLE_TEST_AND_UPDATE", - "task_id", *new_task_args[:3]) - check_task_reply(response, task_args[1:], updated=False) - # Check that the update did not happen. - get_response = self.redis.execute_command("RAY.TASK_TABLE_GET", - "task_id") - self.assertNotEqual(get_response, old_response) - check_task_reply(get_response, task_args[1:]) - - def check_task_subscription(self, p, scheduling_state, local_scheduler_id): - task_args = [ - b"task_id", scheduling_state, - local_scheduler_id.encode("ascii"), b"", 0, b"task_spec" - ] - self.redis.execute_command("RAY.TASK_TABLE_ADD", *task_args) - # Receive the data. - message = get_next_message(p)["data"] - # Check that the notification object is correct. - notification_object = ray.gcs_utils.TaskReply.GetRootAsTaskReply( - message, 0) - self.assertEqual(notification_object.TaskId(), task_args[0]) - self.assertEqual(notification_object.State(), task_args[1]) - self.assertEqual(notification_object.LocalSchedulerId(), task_args[2]) - self.assertEqual(notification_object.ExecutionDependencies(), - task_args[3]) - self.assertEqual(notification_object.TaskSpec(), task_args[-1]) - - def testTaskTableSubscribe(self): - scheduling_state = 1 - local_scheduler_id = "local_scheduler_id" - # Subscribe to the task table. - p = self.redis.pubsub() - p.psubscribe("{prefix}*:*".format(prefix=ray.gcs_utils.TASK_PREFIX)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 1) - self.check_task_subscription(p, scheduling_state, local_scheduler_id) - # unsubscribe to make sure there is only one subscriber at a given time - p.punsubscribe("{prefix}*:*".format(prefix=ray.gcs_utils.TASK_PREFIX)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 0) - - p.psubscribe("{prefix}*:{state}".format( - prefix=ray.gcs_utils.TASK_PREFIX, state=scheduling_state)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 1) - self.check_task_subscription(p, scheduling_state, local_scheduler_id) - p.punsubscribe("{prefix}*:{state}".format( - prefix=ray.gcs_utils.TASK_PREFIX, state=scheduling_state)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 0) - - p.psubscribe("{prefix}{local_scheduler_id}:*".format( - prefix=ray.gcs_utils.TASK_PREFIX, - local_scheduler_id=local_scheduler_id)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 1) - self.check_task_subscription(p, scheduling_state, local_scheduler_id) - p.punsubscribe("{prefix}{local_scheduler_id}:*".format( - prefix=ray.gcs_utils.TASK_PREFIX, - local_scheduler_id=local_scheduler_id)) - # Receive acknowledgment. - self.assertEqual(get_next_message(p)["data"], 0) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/python/ray/common/test/test.py b/python/ray/common/test/test.py deleted file mode 100644 index cd36b697b..000000000 --- a/python/ray/common/test/test.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import pickle -import sys -import unittest - -import ray.local_scheduler as local_scheduler -import ray.ray_constants as ray_constants - - -def random_object_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_function_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_driver_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_task_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -BASE_SIMPLE_OBJECTS = [ - 0, 1, 100000, 0.0, 0.5, 0.9, 100000.1, (), [], {}, "", 990 * "h", u"", - 990 * u"h", - np.ones(3), - np.array([True, False]), None, True, False -] - -if sys.version_info < (3, 0): - BASE_SIMPLE_OBJECTS += [ - long(0), # noqa: E501,F821 - long(1), # noqa: E501,F821 - long(100000), # noqa: E501,F821 - long(1 << 100) # noqa: E501,F821 - ] - -LIST_SIMPLE_OBJECTS = [[obj] for obj in BASE_SIMPLE_OBJECTS] -TUPLE_SIMPLE_OBJECTS = [(obj, ) for obj in BASE_SIMPLE_OBJECTS] -DICT_SIMPLE_OBJECTS = [{(): obj} for obj in BASE_SIMPLE_OBJECTS] - -SIMPLE_OBJECTS = (BASE_SIMPLE_OBJECTS + LIST_SIMPLE_OBJECTS + - TUPLE_SIMPLE_OBJECTS + DICT_SIMPLE_OBJECTS) - -# Create some complex objects that cannot be serialized by value in tasks. - -lst = [] -lst.append(lst) - - -class Foo(object): - def __init__(self): - pass - - -BASE_COMPLEX_OBJECTS = [ - 15000 * "h", 15000 * u"h", lst, - Foo(), 100 * [100 * [10 * [1]]], - np.array([Foo()]) -] - -LIST_COMPLEX_OBJECTS = [[obj] for obj in BASE_COMPLEX_OBJECTS] -TUPLE_COMPLEX_OBJECTS = [(obj, ) for obj in BASE_COMPLEX_OBJECTS] -DICT_COMPLEX_OBJECTS = [{(): obj} for obj in BASE_COMPLEX_OBJECTS] - -COMPLEX_OBJECTS = (BASE_COMPLEX_OBJECTS + LIST_COMPLEX_OBJECTS + - TUPLE_COMPLEX_OBJECTS + DICT_COMPLEX_OBJECTS) - - -class TestSerialization(unittest.TestCase): - def test_serialize_by_value(self): - - for val in SIMPLE_OBJECTS: - self.assertTrue(local_scheduler.check_simple_value(val)) - for val in COMPLEX_OBJECTS: - self.assertFalse(local_scheduler.check_simple_value(val)) - - -class TestObjectID(unittest.TestCase): - def test_create_object_id(self): - random_object_id() - - def test_cannot_pickle_object_ids(self): - object_ids = [random_object_id() for _ in range(256)] - - def f(): - return object_ids - - def g(val=object_ids): - return 1 - - def h(): - object_ids[0] - return 1 - - # Make sure that object IDs cannot be pickled (including functions that - # close over object IDs). - self.assertRaises(Exception, lambda: pickle.dumps(object_ids[0])) - self.assertRaises(Exception, lambda: pickle.dumps(object_ids)) - self.assertRaises(Exception, lambda: pickle.dumps(f)) - self.assertRaises(Exception, lambda: pickle.dumps(g)) - self.assertRaises(Exception, lambda: pickle.dumps(h)) - - def test_equality_comparisons(self): - x1 = local_scheduler.ObjectID(ray_constants.ID_SIZE * b"a") - x2 = local_scheduler.ObjectID(ray_constants.ID_SIZE * b"a") - y1 = local_scheduler.ObjectID(ray_constants.ID_SIZE * b"b") - y2 = local_scheduler.ObjectID(ray_constants.ID_SIZE * b"b") - self.assertEqual(x1, x2) - self.assertEqual(y1, y2) - self.assertNotEqual(x1, y1) - - random_strings = [ - np.random.bytes(ray_constants.ID_SIZE) for _ in range(256) - ] - object_ids1 = [ - local_scheduler.ObjectID(random_strings[i]) for i in range(256) - ] - object_ids2 = [ - local_scheduler.ObjectID(random_strings[i]) for i in range(256) - ] - self.assertEqual(len(set(object_ids1)), 256) - self.assertEqual(len(set(object_ids1 + object_ids2)), 256) - self.assertEqual(set(object_ids1), set(object_ids2)) - - def test_hashability(self): - x = random_object_id() - y = random_object_id() - {x: y} - {x, y} - - -class TestTask(unittest.TestCase): - def check_task(self, task, function_id, num_return_vals, args): - self.assertEqual(function_id.id(), task.function_id().id()) - retrieved_args = task.arguments() - self.assertEqual(num_return_vals, len(task.returns())) - self.assertEqual(len(args), len(retrieved_args)) - for i in range(len(retrieved_args)): - if isinstance(retrieved_args[i], local_scheduler.ObjectID): - self.assertEqual(retrieved_args[i].id(), args[i].id()) - else: - self.assertEqual(retrieved_args[i], args[i]) - - def test_create_and_serialize_task(self): - # TODO(rkn): The function ID should be a FunctionID object, not an - # ObjectID. - driver_id = random_driver_id() - parent_id = random_task_id() - function_id = random_function_id() - object_ids = [random_object_id() for _ in range(256)] - args_list = [[], 1 * [1], 10 * [1], 100 * [1], 1000 * [1], 1 * ["a"], - 10 * ["a"], 100 * ["a"], 1000 * ["a"], [ - 1, 1.3, 2, 1 << 100, "hi", u"hi", [1, 2] - ], object_ids[:1], object_ids[:2], object_ids[:3], - object_ids[:4], object_ids[:5], object_ids[:10], - object_ids[:100], object_ids[:256], [1, object_ids[0]], [ - object_ids[0], "a" - ], [1, object_ids[0], "a"], [ - object_ids[0], 1, object_ids[1], "a" - ], object_ids[:3] + [1, "hi", 2.3] + object_ids[:5], - object_ids + 100 * ["a"] + object_ids] - for args in args_list: - for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: - task = local_scheduler.Task(driver_id, function_id, args, - num_return_vals, parent_id, 0) - self.check_task(task, function_id, num_return_vals, args) - data = local_scheduler.task_to_string(task) - task2 = local_scheduler.task_from_string(data) - self.check_task(task2, function_id, num_return_vals, args) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/python/ray/common/thirdparty/redis/src/.gitkeep b/python/ray/common/thirdparty/redis/src/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/ray/common/__init__.py b/python/ray/core/src/ray/__init__.py similarity index 100% rename from python/ray/common/__init__.py rename to python/ray/core/src/ray/__init__.py diff --git a/python/ray/core/src/local_scheduler/__init__.py b/python/ray/core/src/ray/raylet/__init__.py similarity index 100% rename from python/ray/core/src/local_scheduler/__init__.py rename to python/ray/core/src/ray/raylet/__init__.py diff --git a/python/ray/experimental/sgd/sgd.py b/python/ray/experimental/sgd/sgd.py index c569c036f..82c61613a 100644 --- a/python/ray/experimental/sgd/sgd.py +++ b/python/ray/experimental/sgd/sgd.py @@ -108,8 +108,6 @@ class SGDWorker(object): if plasma_op: store_socket = ( ray.worker.global_worker.plasma_client.store_socket_name) - manager_socket = ( - ray.worker.global_worker.plasma_client.manager_socket_name) if not plasma.tf_plasma_op: plasma.build_plasma_tensorflow_op() @@ -130,7 +128,7 @@ class SGDWorker(object): [grad], self.plasma_in_grads_oids[j], plasma_store_socket_name=store_socket, - plasma_manager_socket_name=manager_socket) + plasma_manager_socket_name="") self.plasma_in_grads.append(plasma_grad) # For applying grads <- plasma @@ -149,7 +147,7 @@ class SGDWorker(object): self.plasma_out_grads_oids[j], dtype=tf.float32, plasma_store_socket_name=store_socket, - plasma_manager_socket_name=manager_socket) + plasma_manager_socket_name="") grad_ph = tf.reshape(grad_ph, self.packed_grads_and_vars[0][j][0].shape) logger.debug("Packed tensor {}".format(grad_ph)) diff --git a/python/ray/experimental/sgd/util.py b/python/ray/experimental/sgd/util.py index ca72bb5e9..6c4f89719 100644 --- a/python/ray/experimental/sgd/util.py +++ b/python/ray/experimental/sgd/util.py @@ -14,15 +14,10 @@ logger = logging.getLogger(__name__) def fetch(oids): - if ray.global_state.use_raylet: - local_sched_client = ray.worker.global_worker.local_scheduler_client - for o in oids: - ray_obj_id = ray.ObjectID(o) - local_sched_client.reconstruct_objects([ray_obj_id], True) - else: - for o in oids: - plasma_id = ray.pyarrow.plasma.ObjectID(o) - ray.worker.global_worker.plasma_client.fetch([plasma_id]) + local_sched_client = ray.worker.global_worker.local_scheduler_client + for o in oids: + ray_obj_id = ray.ObjectID(o) + local_sched_client.reconstruct_objects([ray_obj_id], True) def run_timeline(sess, ops, feed_dict=None, write_timeline=False, name=""): diff --git a/python/ray/experimental/state.py b/python/ray/experimental/state.py index 9f1215a7e..cf3d18283 100644 --- a/python/ray/experimental/state.py +++ b/python/ray/experimental/state.py @@ -6,8 +6,6 @@ import copy from collections import defaultdict import heapq import json -import numbers -import os import redis import sys import time @@ -18,25 +16,6 @@ import ray.ray_constants as ray_constants from ray.utils import (decode, binary_to_object_id, binary_to_hex, hex_to_binary) -# This mapping from integer to task state string must be kept up-to-date with -# the scheduling_state enum in task.h. -TASK_STATUS_WAITING = 1 -TASK_STATUS_SCHEDULED = 2 -TASK_STATUS_QUEUED = 4 -TASK_STATUS_RUNNING = 8 -TASK_STATUS_DONE = 16 -TASK_STATUS_LOST = 32 -TASK_STATUS_RECONSTRUCTING = 64 -TASK_STATUS_MAPPING = { - TASK_STATUS_WAITING: "WAITING", - TASK_STATUS_SCHEDULED: "SCHEDULED", - TASK_STATUS_QUEUED: "QUEUED", - TASK_STATUS_RUNNING: "RUNNING", - TASK_STATUS_DONE: "DONE", - TASK_STATUS_LOST: "LOST", - TASK_STATUS_RECONSTRUCTING: "RECONSTRUCTING", -} - class GlobalState(object): """A class used to interface with the Ray control state. @@ -47,7 +26,6 @@ class GlobalState(object): Attributes: redis_client: The Redis client used to query the primary redis server. redis_clients: Redis clients for each of the Redis shards. - use_raylet: True if we are using the raylet code path. """ def __init__(self): @@ -57,8 +35,6 @@ class GlobalState(object): self.redis_client = None # Clients for the redis shards, storing the object table & task table. self.redis_clients = None - # True if we are using the raylet code path and false otherwise. - self.use_raylet = None def _check_connected(self): """Check that the object has been initialized before it is used. @@ -130,18 +106,6 @@ class GlobalState(object): "ip_address_ports = {}".format( num_redis_shards, ip_address_ports)) - use_raylet = self.redis_client.get("UseRaylet") - if use_raylet is not None: - self.use_raylet = bool(int(use_raylet)) - elif os.environ.get("RAY_USE_XRAY") == "0": - # This environment variable is used in our testing setup. - print("Detected environment variable 'RAY_USE_XRAY' with value " - "{}. This turns OFF xray.".format( - os.environ.get("RAY_USE_XRAY"))) - self.use_raylet = False - else: - self.use_raylet = True - # Get the rest of the information. self.redis_clients = [] for ip_address_port in ip_address_ports: @@ -195,51 +159,23 @@ class GlobalState(object): object_id = ray.ObjectID(hex_to_binary(object_id)) # Return information about a single object ID. - if not self.use_raylet: - # Use the non-raylet code path. - object_locations = self._execute_command( - object_id, "RAY.OBJECT_TABLE_LOOKUP", object_id.id()) - if object_locations is not None: - manager_ids = [ - binary_to_hex(manager_id) - for manager_id in object_locations - ] - else: - manager_ids = None + message = self._execute_command(object_id, "RAY.TABLE_LOOKUP", + ray.gcs_utils.TablePrefix.OBJECT, "", + object_id.id()) + result = [] + gcs_entry = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( + message, 0) - result_table_response = self._execute_command( - object_id, "RAY.RESULT_TABLE_LOOKUP", object_id.id()) - result_table_message = ( - ray.gcs_utils.ResultTableReply.GetRootAsResultTableReply( - result_table_response, 0)) - - result = { - "ManagerIDs": manager_ids, - "TaskID": binary_to_hex(result_table_message.TaskId()), - "IsPut": bool(result_table_message.IsPut()), - "DataSize": result_table_message.DataSize(), - "Hash": binary_to_hex(result_table_message.Hash()) + for i in range(gcs_entry.EntriesLength()): + entry = ray.gcs_utils.ObjectTableData.GetRootAsObjectTableData( + gcs_entry.Entries(i), 0) + object_info = { + "DataSize": entry.ObjectSize(), + "Manager": entry.Manager(), + "IsEviction": entry.IsEviction(), + "NumEvictions": entry.NumEvictions() } - - else: - # Use the raylet code path. - message = self._execute_command(object_id, "RAY.TABLE_LOOKUP", - ray.gcs_utils.TablePrefix.OBJECT, - "", object_id.id()) - result = [] - gcs_entry = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( - message, 0) - - for i in range(gcs_entry.EntriesLength()): - entry = ray.gcs_utils.ObjectTableData.GetRootAsObjectTableData( - gcs_entry.Entries(i), 0) - object_info = { - "DataSize": entry.ObjectSize(), - "Manager": entry.Manager(), - "IsEviction": entry.IsEviction(), - "NumEvictions": entry.NumEvictions() - } - result.append(object_info) + result.append(object_info) return result @@ -259,25 +195,12 @@ class GlobalState(object): return self._object_table(object_id) else: # Return the entire object table. - if not self.use_raylet: - object_info_keys = self._keys( - ray.gcs_utils.OBJECT_INFO_PREFIX + "*") - object_location_keys = self._keys( - ray.gcs_utils.OBJECT_LOCATION_PREFIX + "*") - object_ids_binary = set([ - key[len(ray.gcs_utils.OBJECT_INFO_PREFIX):] - for key in object_info_keys - ] + [ - key[len(ray.gcs_utils.OBJECT_LOCATION_PREFIX):] - for key in object_location_keys - ]) - else: - object_keys = self._keys( - ray.gcs_utils.TablePrefix_OBJECT_string + "*") - object_ids_binary = { - key[len(ray.gcs_utils.TablePrefix_OBJECT_string):] - for key in object_keys - } + object_keys = self._keys(ray.gcs_utils.TablePrefix_OBJECT_string + + "*") + object_ids_binary = { + key[len(ray.gcs_utils.TablePrefix_OBJECT_string):] + for key in object_keys + } results = {} for object_id_binary in object_ids_binary: @@ -294,21 +217,21 @@ class GlobalState(object): Returns: A dictionary with information about the task ID in question. - TASK_STATUS_MAPPING should be used to parse the "State" field - into a human-readable string. """ - if not self.use_raylet: - # Use the non-raylet code path. - task_table_response = self._execute_command( - task_id, "RAY.TASK_TABLE_GET", task_id.id()) - if task_table_response is None: - raise Exception("There is no entry for task ID {} in the task " - "table.".format(binary_to_hex(task_id.id()))) - task_table_message = ray.gcs_utils.TaskReply.GetRootAsTaskReply( - task_table_response, 0) - task_spec = task_table_message.TaskSpec() - task_spec = ray.local_scheduler.task_from_string(task_spec) + message = self._execute_command(task_id, "RAY.TABLE_LOOKUP", + ray.gcs_utils.TablePrefix.RAYLET_TASK, + "", task_id.id()) + gcs_entries = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( + message, 0) + info = [] + for i in range(gcs_entries.EntriesLength()): + task_table_message = ray.gcs_utils.Task.GetRootAsTask( + gcs_entries.Entries(i), 0) + + execution_spec = task_table_message.TaskExecutionSpec() + task_spec = task_table_message.TaskSpecification() + task_spec = ray.raylet.task_from_string(task_spec) task_spec_info = { "DriverID": binary_to_hex(task_spec.driver_id().id()), "TaskID": binary_to_hex(task_spec.task_id().id()), @@ -326,80 +249,19 @@ class GlobalState(object): "RequiredResources": task_spec.required_resources() } - execution_dependencies_message = ( - ray.gcs_utils.TaskExecutionDependencies. - GetRootAsTaskExecutionDependencies( - task_table_message.ExecutionDependencies(), 0)) - execution_dependencies = [ - ray.ObjectID( - execution_dependencies_message.ExecutionDependencies(i)) - for i in range(execution_dependencies_message. - ExecutionDependenciesLength()) - ] - - # TODO(rkn): The return fields ExecutionDependenciesString and - # ExecutionDependencies are redundant, so we should remove - # ExecutionDependencies. However, it is currently used in - # monitor.py. - - return { - "State": task_table_message.State(), - "LocalSchedulerID": binary_to_hex( - task_table_message.LocalSchedulerId()), - "ExecutionDependenciesString": task_table_message. - ExecutionDependencies(), - "ExecutionDependencies": execution_dependencies, - "SpillbackCount": task_table_message.SpillbackCount(), + info.append({ + "ExecutionSpec": { + "Dependencies": [ + execution_spec.Dependencies(i) + for i in range(execution_spec.DependenciesLength()) + ], + "LastTimestamp": execution_spec.LastTimestamp(), + "NumForwards": execution_spec.NumForwards() + }, "TaskSpec": task_spec_info - } + }) - else: - # Use the raylet code path. - message = self._execute_command( - task_id, "RAY.TABLE_LOOKUP", - ray.gcs_utils.TablePrefix.RAYLET_TASK, "", task_id.id()) - gcs_entries = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( - message, 0) - - info = [] - for i in range(gcs_entries.EntriesLength()): - task_table_message = ray.gcs_utils.Task.GetRootAsTask( - gcs_entries.Entries(i), 0) - - execution_spec = task_table_message.TaskExecutionSpec() - task_spec = task_table_message.TaskSpecification() - task_spec = ray.local_scheduler.task_from_string(task_spec) - task_spec_info = { - "DriverID": binary_to_hex(task_spec.driver_id().id()), - "TaskID": binary_to_hex(task_spec.task_id().id()), - "ParentTaskID": binary_to_hex( - task_spec.parent_task_id().id()), - "ParentCounter": task_spec.parent_counter(), - "ActorID": binary_to_hex(task_spec.actor_id().id()), - "ActorCreationID": binary_to_hex( - task_spec.actor_creation_id().id()), - "ActorCreationDummyObjectID": binary_to_hex( - task_spec.actor_creation_dummy_object_id().id()), - "ActorCounter": task_spec.actor_counter(), - "FunctionID": binary_to_hex(task_spec.function_id().id()), - "Args": task_spec.arguments(), - "ReturnObjectIDs": task_spec.returns(), - "RequiredResources": task_spec.required_resources() - } - - info.append({ - "ExecutionSpec": { - "Dependencies": [ - execution_spec.Dependencies(i) - for i in range(execution_spec.DependenciesLength()) - ], - "LastTimestamp": execution_spec.LastTimestamp(), - "NumForwards": execution_spec.NumForwards() - }, - "TaskSpec": task_spec_info - }) - - return info + return info def task_table(self, task_id=None): """Fetch and parse the task table information for one or more task IDs. @@ -416,19 +278,12 @@ class GlobalState(object): task_id = ray.ObjectID(hex_to_binary(task_id)) return self._task_table(task_id) else: - if not self.use_raylet: - task_table_keys = self._keys(ray.gcs_utils.TASK_PREFIX + "*") - task_ids_binary = [ - key[len(ray.gcs_utils.TASK_PREFIX):] - for key in task_table_keys - ] - else: - task_table_keys = self._keys( - ray.gcs_utils.TablePrefix_RAYLET_TASK_string + "*") - task_ids_binary = [ - key[len(ray.gcs_utils.TablePrefix_RAYLET_TASK_string):] - for key in task_table_keys - ] + task_table_keys = self._keys( + ray.gcs_utils.TablePrefix_RAYLET_TASK_string + "*") + task_ids_binary = [ + key[len(ray.gcs_utils.TablePrefix_RAYLET_TASK_string):] + for key in task_table_keys + ] results = {} for task_id_binary in task_ids_binary: @@ -464,95 +319,54 @@ class GlobalState(object): Information about the Ray clients in the cluster. """ self._check_connected() - if not self.use_raylet: - db_client_keys = self.redis_client.keys( - ray.gcs_utils.DB_CLIENT_PREFIX + "*") - node_info = {} - for key in db_client_keys: - client_info = self.redis_client.hgetall(key) - node_ip_address = decode(client_info[b"node_ip_address"]) - if node_ip_address not in node_info: - node_info[node_ip_address] = [] - client_info_parsed = {} - assert b"client_type" in client_info - assert b"deleted" in client_info - assert b"ray_client_id" in client_info - for field, value in client_info.items(): - if field == b"node_ip_address": - pass - elif field == b"client_type": - client_info_parsed["ClientType"] = decode(value) - elif field == b"deleted": - client_info_parsed["Deleted"] = bool( - int(decode(value))) - elif field == b"ray_client_id": - client_info_parsed["DBClientID"] = binary_to_hex(value) - elif field == b"manager_address": - client_info_parsed["AuxAddress"] = decode(value) - elif field == b"local_scheduler_socket_name": - client_info_parsed["LocalSchedulerSocketName"] = ( - decode(value)) - elif client_info[b"client_type"] == b"local_scheduler": - # The remaining fields are resource types. - client_info_parsed[decode(field)] = float( - decode(value)) - else: - client_info_parsed[decode(field)] = decode(value) - node_info[node_ip_address].append(client_info_parsed) + NIL_CLIENT_ID = ray_constants.ID_SIZE * b"\xff" + message = self.redis_client.execute_command( + "RAY.TABLE_LOOKUP", ray.gcs_utils.TablePrefix.CLIENT, "", + NIL_CLIENT_ID) - return node_info + # Handle the case where no clients are returned. This should only + # occur potentially immediately after the cluster is started. + if message is None: + return [] - else: - # This is the raylet code path. - NIL_CLIENT_ID = ray_constants.ID_SIZE * b"\xff" - message = self.redis_client.execute_command( - "RAY.TABLE_LOOKUP", ray.gcs_utils.TablePrefix.CLIENT, "", - NIL_CLIENT_ID) + node_info = {} + gcs_entry = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( + message, 0) - # Handle the case where no clients are returned. This should only - # occur potentially immediately after the cluster is started. - if message is None: - return [] + # Since GCS entries are append-only, we override so that + # only the latest entries are kept. + for i in range(gcs_entry.EntriesLength()): + client = (ray.gcs_utils.ClientTableData.GetRootAsClientTableData( + gcs_entry.Entries(i), 0)) - node_info = {} - gcs_entry = ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( - message, 0) + resources = { + decode(client.ResourcesTotalLabel(i)): + client.ResourcesTotalCapacity(i) + for i in range(client.ResourcesTotalLabelLength()) + } + client_id = ray.utils.binary_to_hex(client.ClientId()) - # Since GCS entries are append-only, we override so that - # only the latest entries are kept. - for i in range(gcs_entry.EntriesLength()): - client = ( - ray.gcs_utils.ClientTableData.GetRootAsClientTableData( - gcs_entry.Entries(i), 0)) + # If this client is being removed, then it must + # have previously been inserted, and + # it cannot have previously been removed. + if not client.IsInsertion(): + assert client_id in node_info, "Client removed not found!" + assert node_info[client_id]["IsInsertion"], ( + "Unexpected duplicate removal of client.") - resources = { - decode(client.ResourcesTotalLabel(i)): - client.ResourcesTotalCapacity(i) - for i in range(client.ResourcesTotalLabelLength()) - } - client_id = ray.utils.binary_to_hex(client.ClientId()) - - # If this client is being removed, then it must - # have previously been inserted, and - # it cannot have previously been removed. - if not client.IsInsertion(): - assert client_id in node_info, "Client removed not found!" - assert node_info[client_id]["IsInsertion"], ( - "Unexpected duplicate removal of client.") - - node_info[client_id] = { - "ClientID": client_id, - "IsInsertion": client.IsInsertion(), - "NodeManagerAddress": decode(client.NodeManagerAddress()), - "NodeManagerPort": client.NodeManagerPort(), - "ObjectManagerPort": client.ObjectManagerPort(), - "ObjectStoreSocketName": decode( - client.ObjectStoreSocketName()), - "RayletSocketName": decode(client.RayletSocketName()), - "Resources": resources - } - return list(node_info.values()) + node_info[client_id] = { + "ClientID": client_id, + "IsInsertion": client.IsInsertion(), + "NodeManagerAddress": decode(client.NodeManagerAddress()), + "NodeManagerPort": client.NodeManagerPort(), + "ObjectManagerPort": client.ObjectManagerPort(), + "ObjectStoreSocketName": decode( + client.ObjectStoreSocketName()), + "RayletSocketName": decode(client.RayletSocketName()), + "Resources": resources + } + return list(node_info.values()) def log_files(self): """Fetch and return a dictionary of log file names to outputs. @@ -755,10 +569,6 @@ class GlobalState(object): return profile_events def profile_table(self): - if not self.use_raylet: - raise Exception("This method is only supported in the raylet " - "code path.") - profile_table_keys = self._keys( ray.gcs_utils.TablePrefix_PROFILE_string + "*") component_identifiers_binary = [ @@ -1207,23 +1017,6 @@ class GlobalState(object): info[key] = cur latest_timestamp = cur - def local_schedulers(self): - """Get a list of live local schedulers. - - Returns: - A list of the live local schedulers. - """ - if self.use_raylet: - raise Exception("The local_schedulers() method is deprecated.") - clients = self.client_table() - local_schedulers = [] - for ip_address, client_list in clients.items(): - for client in client_list: - if (client["ClientType"] == "local_scheduler" - and not client["Deleted"]): - local_schedulers.append(client) - return local_schedulers - def workers(self): """Get a dictionary mapping worker ID to worker information.""" worker_keys = self.redis_client.keys("Worker*") @@ -1237,8 +1030,6 @@ class GlobalState(object): "local_scheduler_socket": (decode( worker_info[b"local_scheduler_socket"])), "node_ip_address": decode(worker_info[b"node_ip_address"]), - "plasma_manager_socket": decode( - worker_info[b"plasma_manager_socket"]), "plasma_store_socket": decode( worker_info[b"plasma_store_socket"]) } @@ -1298,24 +1089,12 @@ class GlobalState(object): resource in the cluster. """ resources = defaultdict(int) - if not self.use_raylet: - local_schedulers = self.local_schedulers() - - for local_scheduler in local_schedulers: - for key, value in local_scheduler.items(): - if key not in [ - "ClientType", "Deleted", "DBClientID", - "AuxAddress", "LocalSchedulerSocketName" - ]: - resources[key] += value - - else: - clients = self.client_table() - for client in clients: - # Only count resources from live clients. - if client["IsInsertion"]: - for key, value in client["Resources"].items(): - resources[key] += value + clients = self.client_table() + for client in clients: + # Only count resources from live clients. + if client["IsInsertion"]: + for key, value in client["Resources"].items(): + resources[key] += value return dict(resources) @@ -1340,93 +1119,48 @@ class GlobalState(object): """ available_resources_by_id = {} - if not self.use_raylet: - subscribe_client = self.redis_client.pubsub() - subscribe_client.subscribe( - ray.gcs_utils.LOCAL_SCHEDULER_INFO_CHANNEL) + subscribe_clients = [ + redis_client.pubsub(ignore_subscribe_messages=True) + for redis_client in self.redis_clients + ] + for subscribe_client in subscribe_clients: + subscribe_client.subscribe(ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL) - local_scheduler_ids = { - local_scheduler["DBClientID"] - for local_scheduler in self.local_schedulers() - } + client_ids = self._live_client_ids() - while set(available_resources_by_id.keys()) != local_scheduler_ids: + while set(available_resources_by_id.keys()) != client_ids: + for subscribe_client in subscribe_clients: + # Parse client message raw_message = subscribe_client.get_message() - if raw_message is None: + if (raw_message is None or raw_message["channel"] != + ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL): continue data = raw_message["data"] - # Ignore subscribtion success message from Redis - # This is a long in python 2 and an int in python 3 - if isinstance(data, numbers.Number): - continue - message = (ray.gcs_utils.LocalSchedulerInfoMessage. - GetRootAsLocalSchedulerInfoMessage(data, 0)) - num_resources = message.DynamicResourcesLength() + gcs_entries = ( + ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( + data, 0)) + heartbeat_data = gcs_entries.Entries(0) + message = (ray.gcs_utils.HeartbeatTableData. + GetRootAsHeartbeatTableData(heartbeat_data, 0)) + # Calculate available resources for this client + num_resources = message.ResourcesAvailableLabelLength() dynamic_resources = {} for i in range(num_resources): - dyn = message.DynamicResources(i) - resource_id = decode(dyn.Key()) - dynamic_resources[resource_id] = dyn.Value() + resource_id = decode(message.ResourcesAvailableLabel(i)) + dynamic_resources[resource_id] = ( + message.ResourcesAvailableCapacity(i)) - # Update available resources for this local scheduler - client_id = binary_to_hex(message.DbClientId()) + # Update available resources for this client + client_id = ray.utils.binary_to_hex(message.ClientId()) available_resources_by_id[client_id] = dynamic_resources - # Update local schedulers in cluster - local_scheduler_ids = { - local_scheduler["DBClientID"] - for local_scheduler in self.local_schedulers() - } - - # Remove disconnected local schedulers - for local_scheduler_id in available_resources_by_id.keys(): - if local_scheduler_id not in local_scheduler_ids: - del available_resources_by_id[local_scheduler_id] - else: - subscribe_clients = [ - redis_client.pubsub(ignore_subscribe_messages=True) - for redis_client in self.redis_clients - ] - for subscribe_client in subscribe_clients: - subscribe_client.subscribe( - ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL) - + # Update clients in cluster client_ids = self._live_client_ids() - while set(available_resources_by_id.keys()) != client_ids: - for subscribe_client in subscribe_clients: - # Parse client message - raw_message = subscribe_client.get_message() - if (raw_message is None or raw_message["channel"] != - ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL): - continue - data = raw_message["data"] - gcs_entries = ( - ray.gcs_utils.GcsTableEntry.GetRootAsGcsTableEntry( - data, 0)) - heartbeat_data = gcs_entries.Entries(0) - message = (ray.gcs_utils.HeartbeatTableData. - GetRootAsHeartbeatTableData(heartbeat_data, 0)) - # Calculate available resources for this client - num_resources = message.ResourcesAvailableLabelLength() - dynamic_resources = {} - for i in range(num_resources): - resource_id = decode( - message.ResourcesAvailableLabel(i)) - dynamic_resources[resource_id] = ( - message.ResourcesAvailableCapacity(i)) - - # Update available resources for this client - client_id = ray.utils.binary_to_hex(message.ClientId()) - available_resources_by_id[client_id] = dynamic_resources - - # Update clients in cluster - client_ids = self._live_client_ids() - - # Remove disconnected clients - for client_id in available_resources_by_id.keys(): - if client_id not in client_ids: - del available_resources_by_id[client_id] + # Remove disconnected clients + for client_id in available_resources_by_id.keys(): + if client_id not in client_ids: + del available_resources_by_id[client_id] # Calculate total available resources total_available_resources = defaultdict(int) @@ -1479,10 +1213,6 @@ class GlobalState(object): A dictionary mapping job ID to a list of the error messages for that job. """ - if not self.use_raylet: - raise Exception("The error_messages method is only supported in " - "the raylet code path.") - if job_id is not None: return self._error_messages(job_id) diff --git a/python/ray/gcs_utils.py b/python/ray/gcs_utils.py index 2616e064d..bbdbe04cf 100644 --- a/python/ray/gcs_utils.py +++ b/python/ray/gcs_utils.py @@ -4,19 +4,6 @@ from __future__ import print_function import flatbuffers -from ray.core.generated.ResultTableReply import ResultTableReply -from ray.core.generated.SubscribeToNotificationsReply \ - import SubscribeToNotificationsReply -from ray.core.generated.TaskExecutionDependencies import \ - TaskExecutionDependencies -from ray.core.generated.TaskReply import TaskReply -from ray.core.generated.DriverTableMessage import DriverTableMessage -from ray.core.generated.LocalSchedulerInfoMessage import \ - LocalSchedulerInfoMessage -from ray.core.generated.SubscribeToDBClientTableReply import \ - SubscribeToDBClientTableReply -from ray.core.generated.TaskInfo import TaskInfo - import ray.core.generated.ErrorTableData from ray.core.generated.GcsTableEntry import GcsTableEntry @@ -32,29 +19,13 @@ from ray.core.generated.TablePrefix import TablePrefix from ray.core.generated.TablePubsub import TablePubsub __all__ = [ - "SubscribeToNotificationsReply", "ResultTableReply", - "TaskExecutionDependencies", "TaskReply", "DriverTableMessage", - "LocalSchedulerInfoMessage", "SubscribeToDBClientTableReply", "TaskInfo", "GcsTableEntry", "ClientTableData", "ErrorTableData", "HeartbeatTableData", "DriverTableData", "ProfileTableData", "ObjectTableData", "Task", "TablePrefix", "TablePubsub", "construct_error_message" ] -# These prefixes must be kept up-to-date with the definitions in -# ray_redis_module.cc. -DB_CLIENT_PREFIX = "CL:" -TASK_PREFIX = "TT:" -OBJECT_CHANNEL_PREFIX = "OC:" -OBJECT_INFO_PREFIX = "OI:" -OBJECT_LOCATION_PREFIX = "OL:" FUNCTION_PREFIX = "RemoteFunction:" -# These prefixes must be kept up-to-date with the definitions in -# common/state/redis.cc -LOCAL_SCHEDULER_INFO_CHANNEL = b"local_schedulers" -PLASMA_MANAGER_HEARTBEAT_CHANNEL = b"plasma_managers" -DRIVER_DEATH_CHANNEL = b"driver_deaths" - # xray heartbeats XRAY_HEARTBEAT_CHANNEL = str(TablePubsub.HEARTBEAT).encode("ascii") diff --git a/python/ray/global_scheduler/__init__.py b/python/ray/global_scheduler/__init__.py deleted file mode 100644 index 25e4d2cf6..000000000 --- a/python/ray/global_scheduler/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from .global_scheduler_services import start_global_scheduler - -__all__ = ["start_global_scheduler"] diff --git a/python/ray/global_scheduler/build/.gitkeep b/python/ray/global_scheduler/build/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/ray/global_scheduler/global_scheduler_services.py b/python/ray/global_scheduler/global_scheduler_services.py deleted file mode 100644 index 7e3d019ff..000000000 --- a/python/ray/global_scheduler/global_scheduler_services.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import subprocess -import time - - -def start_global_scheduler(redis_address, - node_ip_address, - use_valgrind=False, - use_profiler=False, - stdout_file=None, - stderr_file=None): - """Start a global scheduler process. - - Args: - redis_address (str): The address of the Redis instance. - node_ip_address: The IP address of the node that this scheduler will - run on. - use_valgrind (bool): True if the global scheduler should be started - inside of valgrind. If this is True, use_profiler must be False. - use_profiler (bool): True if the global scheduler should be started - inside a profiler. If this is True, use_valgrind must be False. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - - Return: - The process ID of the global scheduler process. - """ - if use_valgrind and use_profiler: - raise Exception("Cannot use valgrind and profiler at the same time.") - global_scheduler_executable = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - "../core/src/global_scheduler/global_scheduler") - command = [ - global_scheduler_executable, "-r", redis_address, "-h", node_ip_address - ] - if use_valgrind: - pid = subprocess.Popen( - [ - "valgrind", "--track-origins=yes", "--leak-check=full", - "--show-leak-kinds=all", "--leak-check-heuristics=stdstring", - "--error-exitcode=1" - ] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - elif use_profiler: - pid = subprocess.Popen( - ["valgrind", "--tool=callgrind"] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - else: - pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file) - time.sleep(0.1) - return pid diff --git a/python/ray/global_scheduler/test/test.py b/python/ray/global_scheduler/test/test.py deleted file mode 100644 index 0e262e705..000000000 --- a/python/ray/global_scheduler/test/test.py +++ /dev/null @@ -1,332 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import os -import random -import signal -import sys -import time -import unittest - -# The ray import must come before the pyarrow import because ray modifies the -# python path so that the right version of pyarrow is found. -import ray.global_scheduler as global_scheduler -import ray.local_scheduler as local_scheduler -import ray.plasma as plasma -from ray.plasma.utils import create_object -from ray import services -from ray.experimental import state -import ray.ray_constants as ray_constants -import pyarrow as pa - -USE_VALGRIND = False -PLASMA_STORE_MEMORY = 1000000000 -NUM_CLUSTER_NODES = 2 - -NIL_WORKER_ID = ray_constants.ID_SIZE * b"\xff" -NIL_OBJECT_ID = ray_constants.ID_SIZE * b"\xff" -NIL_ACTOR_ID = ray_constants.ID_SIZE * b"\xff" - - -def random_driver_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_task_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_function_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_object_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def new_port(): - return random.randint(10000, 65535) - - -class TestGlobalScheduler(unittest.TestCase): - def setUp(self): - # Start one Redis server and N pairs of (plasma, local_scheduler) - self.node_ip_address = "127.0.0.1" - redis_address, redis_shards = services.start_redis( - self.node_ip_address, use_raylet=False) - redis_port = services.get_port(redis_address) - time.sleep(0.1) - # Create a client for the global state store. - self.state = state.GlobalState() - self.state._initialize_global_state(self.node_ip_address, redis_port) - - # Start one global scheduler. - self.p1 = global_scheduler.start_global_scheduler( - redis_address, self.node_ip_address, use_valgrind=USE_VALGRIND) - self.plasma_store_pids = [] - self.plasma_manager_pids = [] - self.local_scheduler_pids = [] - self.plasma_clients = [] - self.local_scheduler_clients = [] - - for i in range(NUM_CLUSTER_NODES): - # Start the Plasma store. Plasma store name is randomly generated. - plasma_store_name, p2 = plasma.start_plasma_store() - self.plasma_store_pids.append(p2) - # Start the Plasma manager. - # Assumption: Plasma manager name and port are randomly generated - # by the plasma module. - manager_info = plasma.start_plasma_manager(plasma_store_name, - redis_address) - plasma_manager_name, p3, plasma_manager_port = manager_info - self.plasma_manager_pids.append(p3) - plasma_address = "{}:{}".format(self.node_ip_address, - plasma_manager_port) - plasma_client = pa.plasma.connect(plasma_store_name, - plasma_manager_name, 64) - self.plasma_clients.append(plasma_client) - # Start the local scheduler. - local_scheduler_name, p4 = local_scheduler.start_local_scheduler( - plasma_store_name, - plasma_manager_name=plasma_manager_name, - plasma_address=plasma_address, - redis_address=redis_address, - static_resources={"CPU": 10}) - # Connect to the scheduler. - local_scheduler_client = local_scheduler.LocalSchedulerClient( - local_scheduler_name, NIL_WORKER_ID, False, random_task_id(), - False) - self.local_scheduler_clients.append(local_scheduler_client) - self.local_scheduler_pids.append(p4) - - def tearDown(self): - # Check that the processes are still alive. - self.assertEqual(self.p1.poll(), None) - for p2 in self.plasma_store_pids: - self.assertEqual(p2.poll(), None) - for p3 in self.plasma_manager_pids: - self.assertEqual(p3.poll(), None) - for p4 in self.local_scheduler_pids: - self.assertEqual(p4.poll(), None) - - redis_processes = services.all_processes[ - services.PROCESS_TYPE_REDIS_SERVER] - for redis_process in redis_processes: - self.assertEqual(redis_process.poll(), None) - - # Kill the global scheduler. - if USE_VALGRIND: - self.p1.send_signal(signal.SIGTERM) - self.p1.wait() - if self.p1.returncode != 0: - os._exit(-1) - else: - self.p1.kill() - # Kill local schedulers, plasma managers, and plasma stores. - for p2 in self.local_scheduler_pids: - p2.kill() - for p3 in self.plasma_manager_pids: - p3.kill() - for p4 in self.plasma_store_pids: - p4.kill() - # Kill Redis. In the event that we are using valgrind, this needs to - # happen after we kill the global scheduler. - while redis_processes: - redis_process = redis_processes.pop() - redis_process.kill() - - def get_plasma_manager_id(self): - """Get the db_client_id with client_type equal to plasma_manager. - - Iterates over all the client table keys, gets the db_client_id for the - client with client_type matching plasma_manager. Strips the client - table prefix. TODO(atumanov): write a separate function to get all - plasma manager client IDs. - - Returns: - The db_client_id if one is found and otherwise None. - """ - db_client_id = None - - client_list = self.state.client_table()[self.node_ip_address] - for client in client_list: - if client["ClientType"] == "plasma_manager": - db_client_id = client["DBClientID"] - break - - return db_client_id - - def test_task_default_resources(self): - task1 = local_scheduler.Task( - random_driver_id(), random_function_id(), [random_object_id()], 0, - random_task_id(), 0) - self.assertEqual(task1.required_resources(), {"CPU": 1}) - task2 = local_scheduler.Task( - random_driver_id(), random_function_id(), [random_object_id()], 0, - random_task_id(), 0, local_scheduler.ObjectID(NIL_ACTOR_ID), - local_scheduler.ObjectID(NIL_OBJECT_ID), - local_scheduler.ObjectID(NIL_ACTOR_ID), - local_scheduler.ObjectID(NIL_ACTOR_ID), 0, 0, [], { - "CPU": 1, - "GPU": 2 - }) - self.assertEqual(task2.required_resources(), {"CPU": 1, "GPU": 2}) - - def test_redis_only_single_task(self): - # Tests global scheduler functionality by interacting with Redis and - # checking task state transitions in Redis only. TODO(atumanov): - # implement. - - # Check precondition for this test: - # There should be 2n+1 db clients: the global scheduler + one local - # scheduler and one plasma per node. - self.assertEqual( - len(self.state.client_table()[self.node_ip_address]), - 2 * NUM_CLUSTER_NODES + 1) - db_client_id = self.get_plasma_manager_id() - assert (db_client_id is not None) - - @unittest.skipIf( - os.environ.get("RAY_USE_NEW_GCS", False), - "New GCS API doesn't have a Python API yet.") - def test_integration_single_task(self): - # There should be three db clients, the global scheduler, the local - # scheduler, and the plasma manager. - self.assertEqual( - len(self.state.client_table()[self.node_ip_address]), - 2 * NUM_CLUSTER_NODES + 1) - - num_return_vals = [0, 1, 2, 3, 5, 10] - # Insert the object into Redis. - data_size = 0xf1f0 - metadata_size = 0x40 - plasma_client = self.plasma_clients[0] - object_dep, memory_buffer, metadata = create_object( - plasma_client, data_size, metadata_size, seal=True) - - # Sleep before submitting task to local scheduler. - time.sleep(0.1) - # Submit a task to Redis. - task = local_scheduler.Task( - random_driver_id(), random_function_id(), - [local_scheduler.ObjectID(object_dep.binary())], - num_return_vals[0], random_task_id(), 0) - self.local_scheduler_clients[0].submit(task) - time.sleep(0.1) - # There should now be a task in Redis, and it should get assigned to - # the local scheduler - num_retries = 10 - while num_retries > 0: - task_entries = self.state.task_table() - self.assertLessEqual(len(task_entries), 1) - if len(task_entries) == 1: - task_id, task = task_entries.popitem() - task_status = task["State"] - self.assertTrue(task_status in [ - state.TASK_STATUS_WAITING, state.TASK_STATUS_SCHEDULED, - state.TASK_STATUS_QUEUED - ]) - if task_status == state.TASK_STATUS_QUEUED: - break - else: - print(task_status) - print("The task has not been scheduled yet, trying again.") - num_retries -= 1 - time.sleep(1) - - if num_retries <= 0 and task_status != state.TASK_STATUS_QUEUED: - # Failed to submit and schedule a single task -- bail. - self.tearDown() - sys.exit(1) - - def integration_many_tasks_helper(self, timesync=True): - # There should be three db clients, the global scheduler, the local - # scheduler, and the plasma manager. - self.assertEqual( - len(self.state.client_table()[self.node_ip_address]), - 2 * NUM_CLUSTER_NODES + 1) - num_return_vals = [0, 1, 2, 3, 5, 10] - - # Submit a bunch of tasks to Redis. - num_tasks = 1000 - for _ in range(num_tasks): - # Create a new object for each task. - data_size = np.random.randint(1 << 12) - metadata_size = np.random.randint(1 << 9) - plasma_client = self.plasma_clients[0] - object_dep, memory_buffer, metadata = create_object( - plasma_client, data_size, metadata_size, seal=True) - if timesync: - # Give 10ms for object info handler to fire (long enough to - # yield CPU). - time.sleep(0.010) - task = local_scheduler.Task( - random_driver_id(), random_function_id(), - [local_scheduler.ObjectID(object_dep.binary())], - num_return_vals[0], random_task_id(), 0) - self.local_scheduler_clients[0].submit(task) - # Check that there are the correct number of tasks in Redis and that - # they all get assigned to the local scheduler. - num_retries = 20 - num_tasks_done = 0 - while num_retries > 0: - task_entries = self.state.task_table() - self.assertLessEqual(len(task_entries), num_tasks) - # First, check if all tasks made it to Redis. - if len(task_entries) == num_tasks: - task_statuses = [ - task_entry["State"] - for task_entry in task_entries.values() - ] - self.assertTrue( - all(status in [ - state.TASK_STATUS_WAITING, state.TASK_STATUS_SCHEDULED, - state.TASK_STATUS_QUEUED - ] for status in task_statuses)) - num_tasks_done = task_statuses.count(state.TASK_STATUS_QUEUED) - num_tasks_scheduled = task_statuses.count( - state.TASK_STATUS_SCHEDULED) - num_tasks_waiting = task_statuses.count( - state.TASK_STATUS_WAITING) - print("tasks in Redis = {}, tasks waiting = {}, " - "tasks scheduled = {}, " - "tasks queued = {}, retries left = {}".format( - len(task_entries), num_tasks_waiting, - num_tasks_scheduled, num_tasks_done, num_retries)) - if all(status == state.TASK_STATUS_QUEUED - for status in task_statuses): - # We're done, so pass. - break - num_retries -= 1 - time.sleep(0.1) - - # Tasks can either be queued or in the global scheduler due to - # spillback. - self.assertEqual(num_tasks_done + num_tasks_waiting, num_tasks) - - @unittest.skipIf( - os.environ.get("RAY_USE_NEW_GCS", False), - "New GCS API doesn't have a Python API yet.") - def test_integration_many_tasks_handler_sync(self): - self.integration_many_tasks_helper(timesync=True) - - @unittest.skipIf( - os.environ.get("RAY_USE_NEW_GCS", False), - "New GCS API doesn't have a Python API yet.") - def test_integration_many_tasks(self): - # More realistic case: should handle out of order object and task - # notifications. - self.integration_many_tasks_helper(timesync=False) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - # Pop the argument so we don't mess with unittest's own argument - # parser. - if sys.argv[-1] == "valgrind": - arg = sys.argv.pop() - USE_VALGRIND = True - print("Using valgrind for tests") - unittest.main(verbosity=2) diff --git a/python/ray/internal/internal_api.py b/python/ray/internal/internal_api.py index 062d633ee..777297431 100644 --- a/python/ray/internal/internal_api.py +++ b/python/ray/internal/internal_api.py @@ -2,7 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import ray.local_scheduler +import ray.raylet import ray.worker from ray import profiling @@ -42,7 +42,4 @@ def free(object_ids, local_only=False, worker=None): if len(object_ids) == 0: return - if worker.use_raylet: - worker.local_scheduler_client.free(object_ids, local_only) - else: - raise Exception("Free is not supported in legacy backend.") + worker.local_scheduler_client.free(object_ids, local_only) diff --git a/python/ray/local_scheduler/build/.gitkeep b/python/ray/local_scheduler/build/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/ray/local_scheduler/local_scheduler_services.py b/python/ray/local_scheduler/local_scheduler_services.py deleted file mode 100644 index c576014e2..000000000 --- a/python/ray/local_scheduler/local_scheduler_services.py +++ /dev/null @@ -1,132 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import multiprocessing -import os -import subprocess -import sys -import time - -from ray.tempfile_services import (get_local_scheduler_socket_name, - get_temp_root) - - -def start_local_scheduler(plasma_store_name, - plasma_manager_name=None, - worker_path=None, - plasma_address=None, - node_ip_address="127.0.0.1", - redis_address=None, - use_valgrind=False, - use_profiler=False, - stdout_file=None, - stderr_file=None, - static_resources=None, - num_workers=0): - """Start a local scheduler process. - - Args: - plasma_store_name (str): The name of the plasma store socket to connect - to. - plasma_manager_name (str): The name of the plasma manager to connect - to. This does not need to be provided, but if it is, then the Redis - address must be provided as well. - worker_path (str): The path of the worker script to use when the local - scheduler starts up new workers. - plasma_address (str): The address of the plasma manager to connect to. - This is only used by the global scheduler to figure out which - plasma managers are connected to which local schedulers. - node_ip_address (str): The address of the node that this local - scheduler is running on. - redis_address (str): The address of the Redis instance to connect to. - If this is not provided, then the local scheduler will not connect - to Redis. - use_valgrind (bool): True if the local scheduler should be started - inside of valgrind. If this is True, use_profiler must be False. - use_profiler (bool): True if the local scheduler should be started - inside a profiler. If this is True, use_valgrind must be False. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - static_resources: A dictionary specifying the local scheduler's - resource capacities. This maps resource names (strings) to - integers or floats. - num_workers (int): The number of workers that the local scheduler - should start. - - Return: - A tuple of the name of the local scheduler socket and the process ID of - the local scheduler process. - """ - if (plasma_manager_name is None) != (redis_address is None): - raise Exception("If one of the plasma_manager_name and the " - "redis_address is provided, then both must be " - "provided.") - if use_valgrind and use_profiler: - raise Exception("Cannot use valgrind and profiler at the same time.") - local_scheduler_executable = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../core/src/local_scheduler/local_scheduler") - local_scheduler_name = get_local_scheduler_socket_name() - command = [ - local_scheduler_executable, "-s", local_scheduler_name, "-p", - plasma_store_name, "-h", node_ip_address, "-n", - str(num_workers) - ] - if plasma_manager_name is not None: - command += ["-m", plasma_manager_name] - if worker_path is not None: - assert plasma_store_name is not None - assert plasma_manager_name is not None - assert redis_address is not None - start_worker_command = ("{} {} " - "--node-ip-address={} " - "--object-store-name={} " - "--object-store-manager-name={} " - "--local-scheduler-name={} " - "--redis-address={} " - "--temp-dir={}".format( - sys.executable, worker_path, - node_ip_address, plasma_store_name, - plasma_manager_name, local_scheduler_name, - redis_address, get_temp_root())) - command += ["-w", start_worker_command] - if redis_address is not None: - command += ["-r", redis_address] - if plasma_address is not None: - command += ["-a", plasma_address] - if static_resources is not None: - resource_argument = "" - for resource_name, resource_quantity in static_resources.items(): - assert (isinstance(resource_quantity, int) - or isinstance(resource_quantity, float)) - resource_argument = ",".join([ - resource_name + "," + str(resource_quantity) - for resource_name, resource_quantity in static_resources.items() - ]) - else: - resource_argument = "CPU,{}".format(multiprocessing.cpu_count()) - command += ["-c", resource_argument] - - if use_valgrind: - pid = subprocess.Popen( - [ - "valgrind", "--track-origins=yes", "--leak-check=full", - "--show-leak-kinds=all", "--leak-check-heuristics=stdstring", - "--error-exitcode=1" - ] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - elif use_profiler: - pid = subprocess.Popen( - ["valgrind", "--tool=callgrind"] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - else: - pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file) - time.sleep(0.1) - return local_scheduler_name, pid diff --git a/python/ray/local_scheduler/test/test.py b/python/ray/local_scheduler/test/test.py deleted file mode 100644 index b35d609de..000000000 --- a/python/ray/local_scheduler/test/test.py +++ /dev/null @@ -1,206 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import os -import signal -import sys -import threading -import time -import unittest - -import ray.local_scheduler as local_scheduler -import ray.plasma as plasma -import ray.ray_constants as ray_constants -import pyarrow as pa - -USE_VALGRIND = False - -NIL_WORKER_ID = ray_constants.ID_SIZE * b"\xff" - - -def random_object_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_driver_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_task_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def random_function_id(): - return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -class TestLocalSchedulerClient(unittest.TestCase): - def setUp(self): - # Start Plasma store. - plasma_store_name, self.p1 = plasma.start_plasma_store() - self.plasma_client = pa.plasma.connect(plasma_store_name, "", 0) - # Start a local scheduler. - scheduler_name, self.p2 = local_scheduler.start_local_scheduler( - plasma_store_name, use_valgrind=USE_VALGRIND) - # Connect to the scheduler. - self.local_scheduler_client = local_scheduler.LocalSchedulerClient( - scheduler_name, NIL_WORKER_ID, False, random_task_id(), False) - - def tearDown(self): - # Check that the processes are still alive. - self.assertEqual(self.p1.poll(), None) - self.assertEqual(self.p2.poll(), None) - - # Kill Plasma. - self.p1.kill() - # Kill the local scheduler. - if USE_VALGRIND: - self.p2.send_signal(signal.SIGTERM) - self.p2.wait() - if self.p2.returncode != 0: - os._exit(-1) - else: - self.p2.kill() - - def test_submit_and_get_task(self): - function_id = random_function_id() - object_ids = [random_object_id() for i in range(256)] - # Create and seal the objects in the object store so that we can - # schedule all of the subsequent tasks. - for object_id in object_ids: - self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0) - self.plasma_client.seal(pa.plasma.ObjectID(object_id.id())) - # Define some arguments to use for the tasks. - args_list = [[], [{}], [()], 1 * [1], 10 * [1], 100 * [1], 1000 * [1], - 1 * ["a"], 10 * ["a"], 100 * ["a"], 1000 * ["a"], [ - 1, 1.3, 1 << 100, "hi", u"hi", [1, 2] - ], object_ids[:1], object_ids[:2], object_ids[:3], - object_ids[:4], object_ids[:5], object_ids[:10], - object_ids[:100], object_ids[:256], [1, object_ids[0]], [ - object_ids[0], "a" - ], [1, object_ids[0], "a"], [ - object_ids[0], 1, object_ids[1], "a" - ], object_ids[:3] + [1, "hi", 2.3] + object_ids[:5], - object_ids + 100 * ["a"] + object_ids] - - for args in args_list: - for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: - task = local_scheduler.Task(random_driver_id(), function_id, - args, num_return_vals, - random_task_id(), 0) - # Submit a task. - self.local_scheduler_client.submit(task) - # Get the task. - new_task = self.local_scheduler_client.get_task() - self.assertEqual(task.function_id().id(), - new_task.function_id().id()) - retrieved_args = new_task.arguments() - returns = new_task.returns() - self.assertEqual(len(args), len(retrieved_args)) - self.assertEqual(num_return_vals, len(returns)) - for i in range(len(retrieved_args)): - if isinstance(args[i], local_scheduler.ObjectID): - self.assertEqual(args[i].id(), retrieved_args[i].id()) - else: - self.assertEqual(args[i], retrieved_args[i]) - - # Submit all of the tasks. - for args in args_list: - for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: - task = local_scheduler.Task(random_driver_id(), function_id, - args, num_return_vals, - random_task_id(), 0) - self.local_scheduler_client.submit(task) - # Get all of the tasks. - for args in args_list: - for num_return_vals in [0, 1, 2, 3, 5, 10, 100]: - new_task = self.local_scheduler_client.get_task() - - def test_scheduling_when_objects_ready(self): - # Create a task and submit it. - object_id = random_object_id() - task = local_scheduler.Task(random_driver_id(), random_function_id(), - [object_id], 0, random_task_id(), 0) - self.local_scheduler_client.submit(task) - - # Launch a thread to get the task. - def get_task(): - self.local_scheduler_client.get_task() - - t = threading.Thread(target=get_task) - t.start() - # Sleep to give the thread time to call get_task. - time.sleep(0.1) - # Create and seal the object ID in the object store. This should - # trigger a scheduling event. - self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0) - self.plasma_client.seal(pa.plasma.ObjectID(object_id.id())) - # Wait until the thread finishes so that we know the task was - # scheduled. - t.join() - - def test_scheduling_when_objects_evicted(self): - # Create a task with two dependencies and submit it. - object_id1 = random_object_id() - object_id2 = random_object_id() - task = local_scheduler.Task(random_driver_id(), random_function_id(), - [object_id1, object_id2], 0, - random_task_id(), 0) - self.local_scheduler_client.submit(task) - - # Launch a thread to get the task. - def get_task(): - self.local_scheduler_client.get_task() - - t = threading.Thread(target=get_task) - t.start() - - # Make one of the dependencies available. - buf = self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1) - self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id())) - # Release the object. - del buf - # Check that the thread is still waiting for a task. - time.sleep(0.1) - self.assertTrue(t.is_alive()) - # Force eviction of the first dependency. - self.plasma_client.evict(plasma.DEFAULT_PLASMA_STORE_MEMORY) - # Check that the thread is still waiting for a task. - time.sleep(0.1) - self.assertTrue(t.is_alive()) - # Check that the first object dependency was evicted. - object1 = self.plasma_client.get_buffers( - [pa.plasma.ObjectID(object_id1.id())], timeout_ms=0) - self.assertEqual(object1, [None]) - # Check that the thread is still waiting for a task. - time.sleep(0.1) - self.assertTrue(t.is_alive()) - - # Create the second dependency. - self.plasma_client.create(pa.plasma.ObjectID(object_id2.id()), 1) - self.plasma_client.seal(pa.plasma.ObjectID(object_id2.id())) - # Check that the thread is still waiting for a task. - time.sleep(0.1) - self.assertTrue(t.is_alive()) - - # Create the first dependency again. Both dependencies are now - # available. - self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1) - self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id())) - - # Wait until the thread finishes so that we know the task was - # scheduled. - t.join() - - -if __name__ == "__main__": - if len(sys.argv) > 1: - # Pop the argument so we don't mess with unittest's own argument - # parser. - if sys.argv[-1] == "valgrind": - arg = sys.argv.pop() - USE_VALGRIND = True - print("Using valgrind for tests") - unittest.main(verbosity=2) diff --git a/python/ray/monitor.py b/python/ray/monitor.py index 6212de23e..8094e3d5e 100644 --- a/python/ray/monitor.py +++ b/python/ray/monitor.py @@ -3,11 +3,9 @@ from __future__ import division from __future__ import print_function import argparse -import binascii import logging import os import time -from collections import Counter, defaultdict import traceback import redis @@ -20,27 +18,6 @@ import ray.utils import ray.ray_constants as ray_constants from ray.services import get_ip_address, get_port from ray.utils import binary_to_hex, binary_to_object_id, hex_to_binary -from ray.worker import NIL_ACTOR_ID - -# These variables must be kept in sync with the C codebase. -# common/common.h -NIL_ID = b"\xff" * ray_constants.ID_SIZE - -# common/task.h -TASK_STATUS_LOST = 32 - -# common/redis_module/ray_redis_module.cc -OBJECT_INFO_PREFIX = b"OI:" -OBJECT_LOCATION_PREFIX = b"OL:" -TASK_TABLE_PREFIX = b"TT:" -DB_CLIENT_PREFIX = b"CL:" -DB_CLIENT_TABLE_NAME = b"db_clients" - -# local_scheduler/local_scheduler.h -LOCAL_SCHEDULER_CLIENT_TYPE = b"local_scheduler" - -# plasma/plasma_manager.cc -PLASMA_MANAGER_CLIENT_TYPE = b"plasma_manager" # Set up logging. logger = logging.getLogger(__name__) @@ -55,19 +32,8 @@ class Monitor(object): Attributes: redis: A connection to the Redis server. - use_raylet: A bool indicating whether to use the raylet code path or - not. subscribe_client: A pubsub client for the Redis server. This is used to receive notifications about failed components. - dead_local_schedulers: A set of the local scheduler IDs of all of the - local schedulers that were up at one point and have died since - then. - live_plasma_managers: A counter mapping live plasma manager IDs to the - number of heartbeats that have passed since we last heard from that - plasma manager. A plasma manager is live if we received a heartbeat - from it at any point, and if it has not timed out. - dead_plasma_managers: A set of the plasma manager IDs of all the plasma - managers that were up at one point and have died since then. """ def __init__(self, @@ -79,26 +45,16 @@ class Monitor(object): self.state = ray.experimental.state.GlobalState() self.state._initialize_global_state( redis_address, redis_port, redis_password=redis_password) - self.use_raylet = self.state.use_raylet self.redis = redis.StrictRedis( host=redis_address, port=redis_port, db=0, password=redis_password) # Setup subscriptions to the primary Redis server and the Redis shards. self.primary_subscribe_client = self.redis.pubsub( ignore_subscribe_messages=True) - if self.use_raylet: - self.shard_subscribe_clients = [] - for redis_client in self.state.redis_clients: - subscribe_client = redis_client.pubsub( - ignore_subscribe_messages=True) - self.shard_subscribe_clients.append(subscribe_client) - else: - # We don't need to subscribe to the shards in legacy Ray. - self.shard_subscribe_clients = [] - # Initialize data structures to keep track of the active database - # clients. - self.dead_local_schedulers = set() - self.live_plasma_managers = Counter() - self.dead_plasma_managers = set() + self.shard_subscribe_clients = [] + for redis_client in self.state.redis_clients: + subscribe_client = redis_client.pubsub( + ignore_subscribe_messages=True) + self.shard_subscribe_clients.append(subscribe_client) # Keep a mapping from local scheduler client ID to IP address to use # for updating the load metrics. self.local_scheduler_id_to_ip_map = {} @@ -152,170 +108,6 @@ class Monitor(object): for subscribe_client in self.shard_subscribe_clients: subscribe_client.subscribe(channel) - def cleanup_task_table(self): - """Clean up global state for failed local schedulers. - - This marks any tasks that were scheduled on dead local schedulers as - TASK_STATUS_LOST. A local scheduler is deemed dead if it is in - self.dead_local_schedulers. - """ - tasks = self.state.task_table() - num_tasks_updated = 0 - for task_id, task in tasks.items(): - # See if the corresponding local scheduler is alive. - if task["LocalSchedulerID"] not in self.dead_local_schedulers: - continue - - # Remove dummy objects returned by actor tasks from any plasma - # manager. Although the objects may still exist in that object - # store, this deletion makes them effectively unreachable by any - # local scheduler connected to a different store. - # TODO(swang): Actually remove the objects from the object store, - # so that the reconstructed actor can reuse the same object store. - if hex_to_binary(task["TaskSpec"]["ActorID"]) != NIL_ACTOR_ID: - dummy_object_id = task["TaskSpec"]["ReturnObjectIDs"][-1] - obj = self.state.object_table(dummy_object_id) - manager_ids = obj["ManagerIDs"] - if manager_ids is not None: - # The dummy object should exist on at most one plasma - # manager, the manager associated with the local scheduler - # that died. - assert len(manager_ids) <= 1 - # Remove the dummy object from the plasma manager - # associated with the dead local scheduler, if any. - for manager in manager_ids: - ok = self.state._execute_command( - dummy_object_id, "RAY.OBJECT_TABLE_REMOVE", - dummy_object_id.id(), hex_to_binary(manager)) - if ok != b"OK": - logger.warn("Failed to remove object location for " - "dead plasma manager.") - - # If the task is scheduled on a dead local scheduler, mark the - # task as lost. - key = binary_to_object_id(hex_to_binary(task_id)) - ok = self.state._execute_command( - key, "RAY.TASK_TABLE_UPDATE", hex_to_binary(task_id), - ray.experimental.state.TASK_STATUS_LOST, NIL_ID, - task["ExecutionDependenciesString"], task["SpillbackCount"]) - if ok != b"OK": - logger.warn("Failed to update lost task for dead scheduler.") - num_tasks_updated += 1 - - if num_tasks_updated > 0: - logger.warn("Marked {} tasks as lost.".format(num_tasks_updated)) - - def cleanup_object_table(self): - """Clean up global state for failed plasma managers. - - This removes dead plasma managers from any location entries in the - object table. A plasma manager is deemed dead if it is in - self.dead_plasma_managers. - """ - # TODO(swang): Also kill the associated plasma store, since it's no - # longer reachable without a plasma manager. - objects = self.state.object_table() - num_objects_removed = 0 - for object_id, obj in objects.items(): - manager_ids = obj["ManagerIDs"] - if manager_ids is None: - continue - for manager in manager_ids: - if manager in self.dead_plasma_managers: - # If the object was on a dead plasma manager, remove that - # location entry. - ok = self.state._execute_command( - object_id, "RAY.OBJECT_TABLE_REMOVE", object_id.id(), - hex_to_binary(manager)) - if ok != b"OK": - logger.warn("Failed to remove object location for " - "dead plasma manager.") - num_objects_removed += 1 - if num_objects_removed > 0: - logger.warn("Marked {} objects as lost." - .format(num_objects_removed)) - - def scan_db_client_table(self): - """Scan the database client table for dead clients. - - After subscribing to the client table, it's necessary to call this - before reading any messages from the subscription channel. This ensures - that we do not miss any notifications for deleted clients that occurred - before we subscribed. - """ - # Exit if we are using the raylet code path because client_table is - # implemented differently. TODO(rkn): Fix this. - if self.use_raylet: - return - - clients = self.state.client_table() - for node_ip_address, node_clients in clients.items(): - for client in node_clients: - db_client_id = client["DBClientID"] - client_type = client["ClientType"] - if client["Deleted"]: - if client_type == LOCAL_SCHEDULER_CLIENT_TYPE: - self.dead_local_schedulers.add(db_client_id) - elif client_type == PLASMA_MANAGER_CLIENT_TYPE: - self.dead_plasma_managers.add(db_client_id) - - def db_client_notification_handler(self, unused_channel, data): - """Handle a notification from the db_client table from Redis. - - This handler processes notifications from the db_client table. - Notifications should be parsed using the SubscribeToDBClientTableReply - flatbuffer. Deletions are processed, insertions are ignored. Cleanup of - the associated state in the state tables should be handled by the - caller. - """ - notification_object = (ray.gcs_utils.SubscribeToDBClientTableReply. - GetRootAsSubscribeToDBClientTableReply(data, 0)) - db_client_id = binary_to_hex(notification_object.DbClientId()) - client_type = notification_object.ClientType() - is_insertion = notification_object.IsInsertion() - - # If the update was an insertion, we ignore it. - if is_insertion: - return - - # If the update was a deletion, add them to our accounting for dead - # local schedulers and plasma managers. - logger.warn("Removed {}, client ID {}".format(client_type, - db_client_id)) - if client_type == LOCAL_SCHEDULER_CLIENT_TYPE: - if db_client_id not in self.dead_local_schedulers: - self.dead_local_schedulers.add(db_client_id) - elif client_type == PLASMA_MANAGER_CLIENT_TYPE: - if db_client_id not in self.dead_plasma_managers: - self.dead_plasma_managers.add(db_client_id) - # Stop tracking this plasma manager's heartbeats, since it's - # already dead. - del self.live_plasma_managers[db_client_id] - - def local_scheduler_info_handler(self, unused_channel, data): - """Handle a local scheduler heartbeat from Redis.""" - - message = (ray.gcs_utils.LocalSchedulerInfoMessage. - GetRootAsLocalSchedulerInfoMessage(data, 0)) - num_resources = message.DynamicResourcesLength() - static_resources = {} - dynamic_resources = {} - for i in range(num_resources): - dyn = message.DynamicResources(i) - static = message.StaticResources(i) - dynamic_resources[dyn.Key().decode("utf-8")] = dyn.Value() - static_resources[static.Key().decode("utf-8")] = static.Value() - - # Update the load metrics for this local scheduler. - client_id = binascii.hexlify(message.DbClientId()).decode("utf-8") - ip = self.local_scheduler_id_to_ip_map.get(client_id) - if ip: - self.load_metrics.update(ip, static_resources, dynamic_resources) - else: - logger.warning( - "Warning: could not find ip for client {} in {}.".format( - client_id, self.local_scheduler_id_to_ip_map)) - def xray_heartbeat_handler(self, unused_channel, data): """Handle an xray heartbeat message from Redis.""" @@ -342,160 +134,6 @@ class Monitor(object): print("Warning: could not find ip for client {} in {}.".format( client_id, self.local_scheduler_id_to_ip_map)) - def plasma_manager_heartbeat_handler(self, unused_channel, data): - """Handle a plasma manager heartbeat from Redis. - - This resets the number of heartbeats that we've missed from this plasma - manager. - """ - # The first ray_constants.ID_SIZE characters are the client ID. - db_client_id = data[:ray_constants.ID_SIZE] - # Reset the number of heartbeats that we've missed from this plasma - # manager. - self.live_plasma_managers[db_client_id] = 0 - - def _entries_for_driver_in_shard(self, driver_id, redis_shard_index): - """Collect IDs of control-state entries for a driver from a shard. - - Args: - driver_id: The ID of the driver. - redis_shard_index: The index of the Redis shard to query. - - Returns: - Lists of IDs: (returned_object_ids, task_ids, put_objects). The - first two are relevant to the driver and are safe to delete. - The last contains all "put" objects in this redis shard; each - element is an (object_id, corresponding task_id) pair. - """ - # TODO(zongheng): consider adding save & restore functionalities. - redis = self.state.redis_clients[redis_shard_index] - task_table_infos = {} # task id -> TaskInfo messages - - # Scan the task table & filter to get the list of tasks belong to this - # driver. Use a cursor in order not to block the redis shards. - for key in redis.scan_iter(match=TASK_TABLE_PREFIX + b"*"): - entry = redis.hgetall(key) - task_info = ray.gcs_utils.TaskInfo.GetRootAsTaskInfo( - entry[b"TaskSpec"], 0) - if driver_id != task_info.DriverId(): - # Ignore tasks that aren't from this driver. - continue - task_table_infos[task_info.TaskId()] = task_info - - # Get the list of objects returned by these tasks. Note these might - # not belong to this redis shard. - returned_object_ids = [] - for task_info in task_table_infos.values(): - returned_object_ids.extend([ - task_info.Returns(i) for i in range(task_info.ReturnsLength()) - ]) - - # Also record all the ray.put()'d objects. - put_objects = [] - for key in redis.scan_iter(match=OBJECT_INFO_PREFIX + b"*"): - entry = redis.hgetall(key) - if entry[b"is_put"] == "0": - continue - object_id = key.split(OBJECT_INFO_PREFIX)[1] - task_id = entry[b"task"] - put_objects.append((object_id, task_id)) - - return returned_object_ids, task_table_infos.keys(), put_objects - - def _clean_up_entries_from_shard(self, object_ids, task_ids, shard_index): - redis = self.state.redis_clients[shard_index] - # Clean up (in the future, save) entries for non-empty objects. - object_ids_locs = set() - object_ids_infos = set() - for object_id in object_ids: - # OL. - obj_loc = redis.zrange(OBJECT_LOCATION_PREFIX + object_id, 0, -1) - if obj_loc: - object_ids_locs.add(object_id) - # OI. - obj_info = redis.hgetall(OBJECT_INFO_PREFIX + object_id) - if obj_info: - object_ids_infos.add(object_id) - - # Form the redis keys to delete. - keys = [TASK_TABLE_PREFIX + k for k in task_ids] - keys.extend([OBJECT_LOCATION_PREFIX + k for k in object_ids_locs]) - keys.extend([OBJECT_INFO_PREFIX + k for k in object_ids_infos]) - - if not keys: - return - # Remove with best effort. - num_deleted = redis.delete(*keys) - logger.info( - "Removed {} dead redis entries of the driver from redis shard {}.". - format(num_deleted, shard_index)) - if num_deleted != len(keys): - logger.warning( - "Failed to remove {} relevant redis entries" - " from redis shard {}.".format(len(keys) - num_deleted)) - - def _clean_up_entries_for_driver(self, driver_id): - """Remove this driver's object/task entries from all redis shards. - - Specifically, removes control-state entries of: - * all objects (OI and OL entries) created by `ray.put()` from the - driver - * all tasks belonging to the driver. - """ - # TODO(zongheng): handle function_table, client_table, log_files -- - # these are in the metadata redis server, not in the shards. - driver_object_ids = [] - driver_task_ids = [] - all_put_objects = [] - - # Collect relevant ids. - # TODO(zongheng): consider parallelizing this loop. - for shard_index in range(len(self.state.redis_clients)): - returned_object_ids, task_ids, put_objects = \ - self._entries_for_driver_in_shard(driver_id, shard_index) - driver_object_ids.extend(returned_object_ids) - driver_task_ids.extend(task_ids) - all_put_objects.extend(put_objects) - - # For the put objects, keep those from relevant tasks. - driver_task_ids_set = set(driver_task_ids) - for object_id, task_id in all_put_objects: - if task_id in driver_task_ids_set: - driver_object_ids.append(object_id) - - # Partition IDs and distribute to shards. - object_ids_per_shard = defaultdict(list) - task_ids_per_shard = defaultdict(list) - - def ToShardIndex(index): - return binary_to_object_id(index).redis_shard_hash() % len( - self.state.redis_clients) - - for object_id in driver_object_ids: - object_ids_per_shard[ToShardIndex(object_id)].append(object_id) - for task_id in driver_task_ids: - task_ids_per_shard[ToShardIndex(task_id)].append(task_id) - - # TODO(zongheng): consider parallelizing this loop. - for shard_index in range(len(self.state.redis_clients)): - self._clean_up_entries_from_shard( - object_ids_per_shard[shard_index], - task_ids_per_shard[shard_index], shard_index) - - def driver_removed_handler(self, unused_channel, data): - """Handle a notification that a driver has been removed. - - This releases any GPU resources that were reserved for that driver in - Redis. - """ - message = ray.gcs_utils.DriverTableMessage.GetRootAsDriverTableMessage( - data, 0) - driver_id = message.DriverId() - logger.info("Driver {} has been removed.".format( - binary_to_hex(driver_id))) - - self._clean_up_entries_for_driver(driver_id) - def _xray_clean_up_entries_for_driver(self, driver_id): """Remove this driver's object/task entries from redis. @@ -529,7 +167,7 @@ class Monitor(object): driver_object_id_bins = set() for object_id, object_table_object in object_table_objects.items(): assert len(object_table_object) > 0 - task_id_bin = ray.local_scheduler.compute_task_id(object_id).id() + task_id_bin = ray.raylet.compute_task_id(object_id).id() if task_id_bin in driver_task_id_bins: driver_object_id_bins.add(object_id.id()) @@ -602,20 +240,7 @@ class Monitor(object): # Determine the appropriate message handler. message_handler = None - if channel == ray.gcs_utils.PLASMA_MANAGER_HEARTBEAT_CHANNEL: - # The message was a heartbeat from a plasma manager. - message_handler = self.plasma_manager_heartbeat_handler - elif channel == ray.gcs_utils.LOCAL_SCHEDULER_INFO_CHANNEL: - # The message was a heartbeat from a local scheduler - message_handler = self.local_scheduler_info_handler - elif channel == DB_CLIENT_TABLE_NAME: - # The message was a notification from the db_client table. - message_handler = self.db_client_notification_handler - elif channel == ray.gcs_utils.DRIVER_DEATH_CHANNEL: - # The message was a notification that a driver was removed. - logger.info("message-handler: driver_removed_handler") - message_handler = self.driver_removed_handler - elif channel == ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL: + if channel == ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL: # Similar functionality as local scheduler info channel message_handler = self.xray_heartbeat_handler elif channel == ray.gcs_utils.XRAY_DRIVER_CHANNEL: @@ -629,10 +254,7 @@ class Monitor(object): message_handler(channel, data) def update_local_scheduler_map(self): - if self.use_raylet: - local_schedulers = self.state.client_table() - else: - local_schedulers = self.state.local_schedulers() + local_schedulers = self.state.client_table() self.local_scheduler_id_to_ip_map = {} for local_scheduler_info in local_schedulers: client_id = local_scheduler_info.get("DBClientID") or \ @@ -680,33 +302,11 @@ class Monitor(object): clients and cleaning up state accordingly. """ # Initialize the subscription channel. - self.subscribe(DB_CLIENT_TABLE_NAME) - self.subscribe(ray.gcs_utils.LOCAL_SCHEDULER_INFO_CHANNEL) - self.subscribe(ray.gcs_utils.PLASMA_MANAGER_HEARTBEAT_CHANNEL) - self.subscribe(ray.gcs_utils.DRIVER_DEATH_CHANNEL) self.subscribe(ray.gcs_utils.XRAY_HEARTBEAT_CHANNEL, primary=False) self.subscribe(ray.gcs_utils.XRAY_DRIVER_CHANNEL) - # Scan the database table for dead database clients. NOTE: This must be - # called before reading any messages from the subscription channel. - # This ensures that we start in a consistent state, since we may have - # missed notifications that were sent before we connected to the - # subscription channel. - self.scan_db_client_table() - # If there were any dead clients at startup, clean up the associated - # state in the state tables. - if len(self.dead_local_schedulers) > 0: - self.cleanup_task_table() - if len(self.dead_plasma_managers) > 0: - self.cleanup_object_table() - - num_plasma_managers = len(self.live_plasma_managers) + len( - self.dead_plasma_managers) - - logger.debug("{} dead local schedulers, {} plasma managers total, {} " - "dead plasma managers".format( - len(self.dead_local_schedulers), num_plasma_managers, - len(self.dead_plasma_managers))) + # TODO(rkn): If there were any dead clients at startup, we should clean + # up the associated state in the state tables. # Handle messages from the subscription channels. while True: @@ -720,43 +320,9 @@ class Monitor(object): self._maybe_flush_gcs() - # Record how many dead local schedulers and plasma managers we had - # at the beginning of this round. - num_dead_local_schedulers = len(self.dead_local_schedulers) - num_dead_plasma_managers = len(self.dead_plasma_managers) - # Process a round of messages. self.process_messages() - # If any new local schedulers or plasma managers were marked as - # dead in this round, clean up the associated state. - if len(self.dead_local_schedulers) > num_dead_local_schedulers: - self.cleanup_task_table() - if len(self.dead_plasma_managers) > num_dead_plasma_managers: - self.cleanup_object_table() - - # Handle plasma managers that timed out during this round. - plasma_manager_ids = list(self.live_plasma_managers.keys()) - for plasma_manager_id in plasma_manager_ids: - if ((self.live_plasma_managers[plasma_manager_id]) >= - ray._config.num_heartbeats_timeout()): - logger.warn("Timed out {}" - .format(PLASMA_MANAGER_CLIENT_TYPE)) - # Remove the plasma manager from the managers whose - # heartbeats we're tracking. - del self.live_plasma_managers[plasma_manager_id] - # Remove the plasma manager from the db_client table. The - # corresponding state in the object table will be cleaned - # up once we receive the notification for this db_client - # deletion. - self.redis.execute_command("RAY.DISCONNECT", - plasma_manager_id) - - # Increment the number of heartbeats that we've missed from each - # plasma manager. - for plasma_manager_id in self.live_plasma_managers: - self.live_plasma_managers[plasma_manager_id] += 1 - # Wait for a heartbeat interval before processing the next round of # messages. time.sleep(ray._config.heartbeat_timeout_milliseconds() * 1e-3) @@ -827,6 +393,5 @@ if __name__ == "__main__": message = "The monitor failed with the following error:\n{}".format( traceback_str) ray.utils.push_error_to_driver_through_redis( - redis_client, monitor.use_raylet, ray_constants.MONITOR_DIED_ERROR, - message) + redis_client, ray_constants.MONITOR_DIED_ERROR, message) raise e diff --git a/python/ray/plasma/__init__.py b/python/ray/plasma/__init__.py index 1ecd0c2af..6c6c18b7c 100644 --- a/python/ray/plasma/__init__.py +++ b/python/ray/plasma/__init__.py @@ -2,9 +2,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from ray.plasma.plasma import (start_plasma_store, start_plasma_manager, - DEFAULT_PLASMA_STORE_MEMORY) +from ray.plasma.plasma import start_plasma_store, DEFAULT_PLASMA_STORE_MEMORY -__all__ = [ - "start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY" -] +__all__ = ["start_plasma_store", "DEFAULT_PLASMA_STORE_MEMORY"] diff --git a/python/ray/plasma/plasma.py b/python/ray/plasma/plasma.py index 262aeebfb..53b243426 100644 --- a/python/ray/plasma/plasma.py +++ b/python/ray/plasma/plasma.py @@ -3,17 +3,13 @@ from __future__ import division from __future__ import print_function import os -import random import subprocess import sys import time -from ray.tempfile_services import (get_object_store_socket_name, - get_plasma_manager_socket_name) +from ray.tempfile_services import get_object_store_socket_name -__all__ = [ - "start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY" -] +__all__ = ["start_plasma_store", "DEFAULT_PLASMA_STORE_MEMORY"] PLASMA_WAIT_TIMEOUT = 2**30 @@ -97,98 +93,3 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file) time.sleep(0.1) return plasma_store_name, pid - - -def new_port(): - return random.randint(10000, 65535) - - -def start_plasma_manager(store_name, - redis_address, - node_ip_address="127.0.0.1", - plasma_manager_port=None, - num_retries=20, - use_valgrind=False, - run_profiler=False, - stdout_file=None, - stderr_file=None): - """Start a plasma manager and return the ports it listens on. - - Args: - store_name (str): The name of the plasma store socket. - redis_address (str): The address of the Redis server. - node_ip_address (str): The IP address of the node. - plasma_manager_port (int): The port to use for the plasma manager. If - this is not provided, a port will be generated at random. - use_valgrind (bool): True if the Plasma manager should be started - inside of valgrind and False otherwise. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - - Returns: - A tuple of the Plasma manager socket name, the process ID of the - Plasma manager process, and the port that the manager is - listening on. - - Raises: - Exception: An exception is raised if the manager could not be started. - """ - plasma_manager_executable = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - "../core/src/plasma/plasma_manager") - plasma_manager_name = get_plasma_manager_socket_name() - if plasma_manager_port is not None: - if num_retries != 1: - raise Exception("num_retries must be 1 if port is specified.") - else: - plasma_manager_port = new_port() - process = None - counter = 0 - while counter < num_retries: - if counter > 0: - print("Plasma manager failed to start, retrying now.") - command = [ - plasma_manager_executable, - "-s", - store_name, - "-m", - plasma_manager_name, - "-h", - node_ip_address, - "-p", - str(plasma_manager_port), - "-r", - redis_address, - ] - if use_valgrind: - process = subprocess.Popen( - [ - "valgrind", "--track-origins=yes", "--leak-check=full", - "--show-leak-kinds=all", "--error-exitcode=1" - ] + command, - stdout=stdout_file, - stderr=stderr_file) - elif run_profiler: - process = subprocess.Popen( - (["valgrind", "--tool=callgrind"] + command), - stdout=stdout_file, - stderr=stderr_file) - else: - process = subprocess.Popen( - command, stdout=stdout_file, stderr=stderr_file) - # This sleep is critical. If the plasma_manager fails to start because - # the port is already in use, then we need it to fail within 0.1 - # seconds. - if use_valgrind: - time.sleep(1) - else: - time.sleep(0.1) - # See if the process has terminated - if process.poll() is None: - return plasma_manager_name, process, plasma_manager_port - # Generate a new port and try again. - plasma_manager_port = new_port() - counter += 1 - raise Exception("Couldn't start plasma manager.") diff --git a/python/ray/plasma/test/test.py b/python/ray/plasma/test/test.py deleted file mode 100644 index bc2418f00..000000000 --- a/python/ray/plasma/test/test.py +++ /dev/null @@ -1,560 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from numpy.testing import assert_equal -import os -import random -import signal -import subprocess -import sys -import threading -import time -import unittest - -# The ray import must come before the pyarrow import because ray modifies the -# python path so that the right version of pyarrow is found. -import ray -from ray.plasma.utils import (random_object_id, create_object_with_id, - create_object) -import ray.ray_constants as ray_constants -from ray import services -import pyarrow as pa -import pyarrow.plasma as plasma - -USE_VALGRIND = False -PLASMA_STORE_MEMORY = 1000000000 - - -def random_name(): - return str(random.randint(0, 99999999)) - - -def assert_get_object_equal(unit_test, - client1, - client2, - object_id, - memory_buffer=None, - metadata=None): - client1_buff = client1.get_buffers([object_id])[0] - client2_buff = client2.get_buffers([object_id])[0] - client1_metadata = client1.get_metadata([object_id])[0] - client2_metadata = client2.get_metadata([object_id])[0] - unit_test.assertEqual(len(client1_buff), len(client2_buff)) - unit_test.assertEqual(len(client1_metadata), len(client2_metadata)) - # Check that the buffers from the two clients are the same. - assert_equal( - np.frombuffer(client1_buff, dtype="uint8"), - np.frombuffer(client2_buff, dtype="uint8")) - # Check that the metadata buffers from the two clients are the same. - assert_equal( - np.frombuffer(client1_metadata, dtype="uint8"), - np.frombuffer(client2_metadata, dtype="uint8")) - # If a reference buffer was provided, check that it is the same as well. - if memory_buffer is not None: - assert_equal( - np.frombuffer(memory_buffer, dtype="uint8"), - np.frombuffer(client1_buff, dtype="uint8")) - # If reference metadata was provided, check that it is the same as well. - if metadata is not None: - assert_equal( - np.frombuffer(metadata, dtype="uint8"), - np.frombuffer(client1_metadata, dtype="uint8")) - - -DEFAULT_PLASMA_STORE_MEMORY = 10**9 - - -def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, - use_valgrind=False, - use_profiler=False, - stdout_file=None, - stderr_file=None): - """Start a plasma store process. - Args: - use_valgrind (bool): True if the plasma store should be started inside - of valgrind. If this is True, use_profiler must be False. - use_profiler (bool): True if the plasma store should be started inside - a profiler. If this is True, use_valgrind must be False. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - Return: - A tuple of the name of the plasma store socket and the process ID of - the plasma store process. - """ - if use_valgrind and use_profiler: - raise Exception("Cannot use valgrind and profiler at the same time.") - plasma_store_executable = os.path.join(pa.__path__[0], - "plasma_store_server") - plasma_store_name = "/tmp/plasma_store{}".format(random_name()) - command = [ - plasma_store_executable, "-s", plasma_store_name, "-m", - str(plasma_store_memory) - ] - if use_valgrind: - pid = subprocess.Popen( - [ - "valgrind", "--track-origins=yes", "--leak-check=full", - "--show-leak-kinds=all", "--leak-check-heuristics=stdstring", - "--error-exitcode=1" - ] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - elif use_profiler: - pid = subprocess.Popen( - ["valgrind", "--tool=callgrind"] + command, - stdout=stdout_file, - stderr=stderr_file) - time.sleep(1.0) - else: - pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file) - time.sleep(0.1) - return plasma_store_name, pid - - -# Plasma client tests were moved into arrow - - -class TestPlasmaManager(unittest.TestCase): - def setUp(self): - # Start two PlasmaStores. - store_name1, self.p2 = start_plasma_store(use_valgrind=USE_VALGRIND) - store_name2, self.p3 = start_plasma_store(use_valgrind=USE_VALGRIND) - # Start a Redis server. - redis_address, _ = services.start_redis("127.0.0.1", use_raylet=False) - # Start two PlasmaManagers. - manager_name1, self.p4, self.port1 = ray.plasma.start_plasma_manager( - store_name1, redis_address, use_valgrind=USE_VALGRIND) - manager_name2, self.p5, self.port2 = ray.plasma.start_plasma_manager( - store_name2, redis_address, use_valgrind=USE_VALGRIND) - # Connect two PlasmaClients. - self.client1 = plasma.connect(store_name1, manager_name1, 64) - self.client2 = plasma.connect(store_name2, manager_name2, 64) - - # Store the processes that will be explicitly killed during tearDown so - # that a test case can remove ones that will be killed during the test. - # NOTE: If this specific order is changed, valgrind will fail. - self.processes_to_kill = [self.p4, self.p5, self.p2, self.p3] - - def tearDown(self): - # Check that the processes are still alive. - for process in self.processes_to_kill: - self.assertEqual(process.poll(), None) - - # Kill the Plasma store and Plasma manager processes. - if USE_VALGRIND: - # Give processes opportunity to finish work. - time.sleep(1) - for process in self.processes_to_kill: - process.send_signal(signal.SIGTERM) - process.wait() - if process.returncode != 0: - print("aborting due to valgrind error") - os._exit(-1) - else: - for process in self.processes_to_kill: - process.kill() - - # Clean up the Redis server. - services.cleanup() - - def test_fetch(self): - for _ in range(10): - # Create an object. - object_id1, memory_buffer1, metadata1 = create_object( - self.client1, 2000, 2000) - self.client1.fetch([object_id1]) - self.assertEqual(self.client1.contains(object_id1), True) - self.assertEqual(self.client2.contains(object_id1), False) - # Fetch the object from the other plasma manager. - # TODO(rkn): Right now we must wait for the object table to be - # updated. - while not self.client2.contains(object_id1): - self.client2.fetch([object_id1]) - # Compare the two buffers. - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id1, - memory_buffer=memory_buffer1, - metadata=metadata1) - - # Test that we can call fetch on object IDs that don't exist yet. - object_id2 = random_object_id() - self.client1.fetch([object_id2]) - self.assertEqual(self.client1.contains(object_id2), False) - memory_buffer2, metadata2 = create_object_with_id( - self.client2, object_id2, 2000, 2000) - # # Check that the object has been fetched. - # self.assertEqual(self.client1.contains(object_id2), True) - # Compare the two buffers. - # assert_get_object_equal(self, self.client1, self.client2, object_id2, - # memory_buffer=memory_buffer2, - # metadata=metadata2) - - # Test calling the same fetch request a bunch of times. - object_id3 = random_object_id() - self.assertEqual(self.client1.contains(object_id3), False) - self.assertEqual(self.client2.contains(object_id3), False) - for _ in range(10): - self.client1.fetch([object_id3]) - self.client2.fetch([object_id3]) - memory_buffer3, metadata3 = create_object_with_id( - self.client1, object_id3, 2000, 2000) - for _ in range(10): - self.client1.fetch([object_id3]) - self.client2.fetch([object_id3]) - # TODO(rkn): Right now we must wait for the object table to be updated. - while not self.client2.contains(object_id3): - self.client2.fetch([object_id3]) - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id3, - memory_buffer=memory_buffer3, - metadata=metadata3) - - def test_fetch_multiple(self): - for _ in range(20): - # Create two objects and a third fake one that doesn't exist. - object_id1, memory_buffer1, metadata1 = create_object( - self.client1, 2000, 2000) - missing_object_id = random_object_id() - object_id2, memory_buffer2, metadata2 = create_object( - self.client1, 2000, 2000) - object_ids = [object_id1, missing_object_id, object_id2] - # Fetch the objects from the other plasma store. The second object - # ID should timeout since it does not exist. - # TODO(rkn): Right now we must wait for the object table to be - # updated. - while ((not self.client2.contains(object_id1)) - or (not self.client2.contains(object_id2))): - self.client2.fetch(object_ids) - # Compare the buffers of the objects that do exist. - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id1, - memory_buffer=memory_buffer1, - metadata=metadata1) - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id2, - memory_buffer=memory_buffer2, - metadata=metadata2) - # Fetch in the other direction. The fake object still does not - # exist. - self.client1.fetch(object_ids) - assert_get_object_equal( - self, - self.client2, - self.client1, - object_id1, - memory_buffer=memory_buffer1, - metadata=metadata1) - assert_get_object_equal( - self, - self.client2, - self.client1, - object_id2, - memory_buffer=memory_buffer2, - metadata=metadata2) - - # Check that we can call fetch with duplicated object IDs. - object_id3 = random_object_id() - self.client1.fetch([object_id3, object_id3]) - object_id4, memory_buffer4, metadata4 = create_object( - self.client1, 2000, 2000) - time.sleep(0.1) - # TODO(rkn): Right now we must wait for the object table to be updated. - while not self.client2.contains(object_id4): - self.client2.fetch( - [object_id3, object_id3, object_id4, object_id4]) - assert_get_object_equal( - self, - self.client2, - self.client1, - object_id4, - memory_buffer=memory_buffer4, - metadata=metadata4) - - def test_wait(self): - # Test timeout. - obj_id0 = random_object_id() - self.client1.wait([obj_id0], timeout=100, num_returns=1) - # If we get here, the test worked. - - # Test wait if local objects available. - obj_id1 = random_object_id() - self.client1.create(obj_id1, 1000) - self.client1.seal(obj_id1) - ready, waiting = self.client1.wait( - [obj_id1], timeout=100, num_returns=1) - self.assertEqual(set(ready), {obj_id1}) - self.assertEqual(waiting, []) - - # Test wait if only one object available and only one object waited - # for. - obj_id2 = random_object_id() - self.client1.create(obj_id2, 1000) - # Don't seal. - ready, waiting = self.client1.wait( - [obj_id2, obj_id1], timeout=100, num_returns=1) - self.assertEqual(set(ready), {obj_id1}) - self.assertEqual(set(waiting), {obj_id2}) - - # Test wait if object is sealed later. - obj_id3 = random_object_id() - - def finish(): - self.client2.create(obj_id3, 1000) - self.client2.seal(obj_id3) - - t = threading.Timer(0.1, finish) - t.start() - ready, waiting = self.client1.wait( - [obj_id3, obj_id2, obj_id1], timeout=1000, num_returns=2) - self.assertEqual(set(ready), {obj_id1, obj_id3}) - self.assertEqual(set(waiting), {obj_id2}) - - # Test if the appropriate number of objects is shown if some objects - # are not ready. - ready, waiting = self.client1.wait([obj_id3, obj_id2, obj_id1], 100, 3) - self.assertEqual(set(ready), {obj_id1, obj_id3}) - self.assertEqual(set(waiting), {obj_id2}) - - # Don't forget to seal obj_id2. - self.client1.seal(obj_id2) - - # Test calling wait a bunch of times. - object_ids = [] - # TODO(rkn): Increasing n to 100 (or larger) will cause failures. The - # problem appears to be that the number of timers added to the manager - # event loop slow down the manager so much that some of the - # asynchronous Redis commands timeout triggering fatal failure - # callbacks. - n = 40 - for i in range(n * (n + 1) // 2): - if i % 2 == 0: - object_id, _, _ = create_object(self.client1, 200, 200) - else: - object_id, _, _ = create_object(self.client2, 200, 200) - object_ids.append(object_id) - # Try waiting for all of the object IDs on the first client. - waiting = object_ids - retrieved = [] - for i in range(1, n + 1): - ready, waiting = self.client1.wait( - waiting, timeout=1000, num_returns=i) - self.assertEqual(len(ready), i) - retrieved += ready - self.assertEqual(set(retrieved), set(object_ids)) - ready, waiting = self.client1.wait( - object_ids, timeout=1000, num_returns=len(object_ids)) - self.assertEqual(set(ready), set(object_ids)) - self.assertEqual(waiting, []) - # Try waiting for all of the object IDs on the second client. - waiting = object_ids - retrieved = [] - for i in range(1, n + 1): - ready, waiting = self.client2.wait( - waiting, timeout=1000, num_returns=i) - self.assertEqual(len(ready), i) - retrieved += ready - self.assertEqual(set(retrieved), set(object_ids)) - ready, waiting = self.client2.wait( - object_ids, timeout=1000, num_returns=len(object_ids)) - self.assertEqual(set(ready), set(object_ids)) - self.assertEqual(waiting, []) - - # Make sure that wait returns when the requested number of object IDs - # are available and does not wait for all object IDs to be available. - object_ids = [random_object_id() for _ in range(9)] + \ - [plasma.ObjectID(ray_constants.ID_SIZE * b'\x00')] - object_ids_perm = object_ids[:] - random.shuffle(object_ids_perm) - for i in range(10): - if i % 2 == 0: - create_object_with_id(self.client1, object_ids_perm[i], 2000, - 2000) - else: - create_object_with_id(self.client2, object_ids_perm[i], 2000, - 2000) - ready, waiting = self.client1.wait(object_ids, num_returns=(i + 1)) - self.assertEqual(set(ready), set(object_ids_perm[:(i + 1)])) - self.assertEqual(set(waiting), set(object_ids_perm[(i + 1):])) - - def test_transfer(self): - num_attempts = 100 - for _ in range(100): - # Create an object. - object_id1, memory_buffer1, metadata1 = create_object( - self.client1, 2000, 2000) - # Transfer the buffer to the the other Plasma store. There is a - # race condition on the create and transfer of the object, so keep - # trying until the object appears on the second Plasma store. - for i in range(num_attempts): - self.client1.transfer("127.0.0.1", self.port2, object_id1) - buff = self.client2.get_buffers( - [object_id1], timeout_ms=100)[0] - if buff is not None: - break - self.assertNotEqual(buff, None) - del buff - - # Compare the two buffers. - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id1, - memory_buffer=memory_buffer1, - metadata=metadata1) - # # Transfer the buffer again. - # self.client1.transfer("127.0.0.1", self.port2, object_id1) - # # Compare the two buffers. - # assert_get_object_equal(self, self.client1, self.client2, - # object_id1, - # memory_buffer=memory_buffer1, - # metadata=metadata1) - - # Create an object. - object_id2, memory_buffer2, metadata2 = create_object( - self.client2, 20000, 20000) - # Transfer the buffer to the the other Plasma store. There is a - # race condition on the create and transfer of the object, so keep - # trying until the object appears on the second Plasma store. - for i in range(num_attempts): - self.client2.transfer("127.0.0.1", self.port1, object_id2) - buff = self.client1.get_buffers( - [object_id2], timeout_ms=100)[0] - if buff is not None: - break - self.assertNotEqual(buff, None) - del buff - - # Compare the two buffers. - assert_get_object_equal( - self, - self.client1, - self.client2, - object_id2, - memory_buffer=memory_buffer2, - metadata=metadata2) - - def test_illegal_functionality(self): - # Create an object id string. - # object_id = random_object_id() - # Create a new buffer. - # memory_buffer = self.client1.create(object_id, 20000) - # This test is commented out because it currently fails. - # # Transferring the buffer before sealing it should fail. - # self.assertRaises(Exception, - # lambda : self.manager1.transfer(1, object_id)) - pass - - def test_stresstest(self): - a = time.time() - object_ids = [] - for i in range(10000): # TODO(pcm): increase this to 100000. - object_id = random_object_id() - object_ids.append(object_id) - self.client1.create(object_id, 1) - self.client1.seal(object_id) - for object_id in object_ids: - self.client1.transfer("127.0.0.1", self.port2, object_id) - b = time.time() - a - - print("it took", b, "seconds to put and transfer the objects") - - -class TestPlasmaManagerRecovery(unittest.TestCase): - def setUp(self): - # Start a Plasma store. - self.store_name, self.p2 = start_plasma_store( - use_valgrind=USE_VALGRIND) - # Start a Redis server. - self.redis_address, _ = services.start_redis( - "127.0.0.1", use_raylet=False) - # Start a PlasmaManagers. - manager_name, self.p3, self.port1 = ray.plasma.start_plasma_manager( - self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) - # Connect a PlasmaClient. - self.client = plasma.connect(self.store_name, manager_name, 64) - - # Store the processes that will be explicitly killed during tearDown so - # that a test case can remove ones that will be killed during the test. - # NOTE: The plasma managers must be killed before the plasma store - # since plasma store death will bring down the managers. - self.processes_to_kill = [self.p3, self.p2] - - def tearDown(self): - # Check that the processes are still alive. - for process in self.processes_to_kill: - self.assertEqual(process.poll(), None) - - # Kill the Plasma store and Plasma manager processes. - if USE_VALGRIND: - # Give processes opportunity to finish work. - time.sleep(1) - for process in self.processes_to_kill: - process.send_signal(signal.SIGTERM) - process.wait() - if process.returncode != 0: - print("aborting due to valgrind error") - os._exit(-1) - else: - for process in self.processes_to_kill: - process.kill() - - # Clean up the Redis server. - services.cleanup() - - def test_delayed_start(self): - num_objects = 10 - # Create some objects using one client. - object_ids = [random_object_id() for _ in range(num_objects)] - for i in range(10): - create_object_with_id(self.client, object_ids[i], 2000, 2000) - - # Wait until the objects have been sealed in the store. - ready, waiting = self.client.wait(object_ids, num_returns=num_objects) - self.assertEqual(set(ready), set(object_ids)) - self.assertEqual(waiting, []) - - # Start a second plasma manager attached to the same store. - manager_name, self.p5, self.port2 = ray.plasma.start_plasma_manager( - self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) - self.processes_to_kill = [self.p5] + self.processes_to_kill - - # Check that the second manager knows about existing objects. - client2 = plasma.connect(self.store_name, manager_name, 64) - ready, waiting = [], object_ids - while True: - ready, waiting = client2.wait( - object_ids, num_returns=num_objects, timeout=0) - if len(ready) == len(object_ids): - break - - self.assertEqual(set(ready), set(object_ids)) - self.assertEqual(waiting, []) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - # Pop the argument so we don't mess with unittest's own argument - # parser. - if sys.argv[-1] == "valgrind": - arg = sys.argv.pop() - USE_VALGRIND = True - print("Using valgrind for tests") - unittest.main(verbosity=2) diff --git a/python/ray/plasma/utils.py b/python/ray/plasma/utils.py deleted file mode 100644 index 45feb0b1d..000000000 --- a/python/ray/plasma/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import random - -import pyarrow.plasma as plasma -import ray.ray_constants as ray_constants - - -def random_object_id(): - return plasma.ObjectID(np.random.bytes(ray_constants.ID_SIZE)) - - -def generate_metadata(length): - metadata_buffer = bytearray(length) - if length > 0: - metadata_buffer[0] = random.randint(0, 255) - metadata_buffer[-1] = random.randint(0, 255) - for _ in range(100): - metadata_buffer[random.randint(0, length - 1)] = (random.randint( - 0, 255)) - return metadata_buffer - - -def write_to_data_buffer(buff, length): - array = np.frombuffer(buff, dtype="uint8") - if length > 0: - array[0] = random.randint(0, 255) - array[-1] = random.randint(0, 255) - for _ in range(100): - array[random.randint(0, length - 1)] = random.randint(0, 255) - - -def create_object_with_id(client, - object_id, - data_size, - metadata_size, - seal=True): - metadata = generate_metadata(metadata_size) - memory_buffer = client.create(object_id, data_size, metadata) - write_to_data_buffer(memory_buffer, data_size) - if seal: - client.seal(object_id) - return memory_buffer, metadata - - -def create_object(client, data_size, metadata_size, seal=True): - object_id = random_object_id() - memory_buffer, metadata = create_object_with_id( - client, object_id, data_size, metadata_size, seal=seal) - return object_id, memory_buffer, metadata diff --git a/python/ray/profiling.py b/python/ray/profiling.py index a16dd9d7a..42b02f892 100644 --- a/python/ray/profiling.py +++ b/python/ray/profiling.py @@ -59,17 +59,7 @@ def profile(event_type, extra_data=None, worker=None): """ if worker is None: worker = ray.worker.global_worker - if not worker.use_raylet: - # Log the event if this is a worker and not a driver, since the - # driver's event log never gets flushed. - if worker.mode == ray.WORKER_MODE: - return RayLogSpanNonRaylet( - worker.profiler, event_type, contents=extra_data) - else: - return NULL_LOG_SPAN - else: - return RayLogSpanRaylet( - worker.profiler, event_type, extra_data=extra_data) + return RayLogSpanRaylet(worker.profiler, event_type, extra_data=extra_data) class Profiler(object): @@ -124,87 +114,20 @@ class Profiler(object): events = self.events self.events = [] - if not self.worker.use_raylet: - event_log_key = b"event_log:" + self.worker.worker_id - event_log_value = json.dumps(events) - self.worker.local_scheduler_client.log_event( - event_log_key, event_log_value, time.time()) + if self.worker.mode == ray.WORKER_MODE: + component_type = "worker" else: - if self.worker.mode == ray.WORKER_MODE: - component_type = "worker" - else: - component_type = "driver" + component_type = "driver" - self.worker.local_scheduler_client.push_profile_events( - component_type, ray.ObjectID(self.worker.worker_id), - self.worker.node_ip_address, events) + self.worker.local_scheduler_client.push_profile_events( + component_type, ray.ObjectID(self.worker.worker_id), + self.worker.node_ip_address, events) def add_event(self, event): with self.lock: self.events.append(event) -class RayLogSpanNonRaylet(object): - """An object used to enable logging a span of events with a with statement. - - Attributes: - event_type (str): The type of the event being logged. - contents: Additional information to log. - """ - - def __init__(self, profiler, event_type, contents=None): - """Initialize a RayLogSpanNonRaylet object.""" - self.profiler = profiler - self.event_type = event_type - self.contents = contents - - def _log(self, event_type, kind, contents=None): - """Log an event to the global state store. - - This adds the event to a buffer of events locally. The buffer can be - flushed and written to the global state store by calling - flush_profile_data(). - - Args: - event_type (str): The type of the event. - contents: More general data to store with the event. - kind (int): Either LOG_POINT, LOG_SPAN_START, or LOG_SPAN_END. This - is LOG_POINT if the event being logged happens at a single - point in time. It is LOG_SPAN_START if we are starting to log a - span of time, and it is LOG_SPAN_END if we are finishing - logging a span of time. - """ - # TODO(rkn): This code currently takes around half a microsecond. Since - # we call it tens of times per task, this adds up. We will need to redo - # the logging code, perhaps in C. - contents = {} if contents is None else contents - assert isinstance(contents, dict) - # Make sure all of the keys and values in the dictionary are strings. - contents = {str(k): str(v) for k, v in contents.items()} - self.profiler.add_event((time.time(), event_type, kind, contents)) - - def __enter__(self): - """Log the beginning of a span event.""" - self._log( - event_type=self.event_type, - contents=self.contents, - kind=LOG_SPAN_START) - - def __exit__(self, type, value, tb): - """Log the end of a span event. Log any exception that occurred.""" - if type is None: - self._log(event_type=self.event_type, kind=LOG_SPAN_END) - else: - self._log( - event_type=self.event_type, - contents={ - "type": str(type), - "value": value, - "traceback": traceback.format_exc() - }, - kind=LOG_SPAN_END) - - class RayLogSpanRaylet(object): """An object used to enable logging a span of events with a with statement. diff --git a/python/ray/ray_constants.py b/python/ray/ray_constants.py index d62b57b5c..a1d5e1a76 100644 --- a/python/ray/ray_constants.py +++ b/python/ray/ray_constants.py @@ -5,7 +5,7 @@ from __future__ import print_function import os -from ray.local_scheduler import ObjectID +from ray.raylet import ObjectID def env_integer(key, default): @@ -41,7 +41,6 @@ REGISTER_ACTOR_PUSH_ERROR = "register_actor" WORKER_CRASH_PUSH_ERROR = "worker_crash" WORKER_DIED_PUSH_ERROR = "worker_died" PUT_RECONSTRUCTION_PUSH_ERROR = "put_reconstruction" -HASH_MISMATCH_PUSH_ERROR = "object_hash_mismatch" INFEASIBLE_TASK_ERROR = "infeasible_task" REMOVED_NODE_ERROR = "node_removed" MONITOR_DIED_ERROR = "monitor_died" diff --git a/python/ray/local_scheduler/__init__.py b/python/ray/raylet/__init__.py similarity index 76% rename from python/ray/local_scheduler/__init__.py rename to python/ray/raylet/__init__.py index a469776f1..8757f5974 100644 --- a/python/ray/local_scheduler/__init__.py +++ b/python/ray/raylet/__init__.py @@ -2,10 +2,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from ray.core.src.local_scheduler.liblocal_scheduler_library_python import ( +from ray.core.src.ray.raylet.liblocal_scheduler_library_python import ( Task, LocalSchedulerClient, ObjectID, check_simple_value, compute_task_id, task_from_string, task_to_string, _config, common_error) -from .local_scheduler_services import start_local_scheduler __all__ = [ "Task", "LocalSchedulerClient", "ObjectID", "check_simple_value", diff --git a/python/ray/rllib/utils/actors.py b/python/ray/rllib/utils/actors.py index 487c3595e..1e19b703d 100644 --- a/python/ray/rllib/utils/actors.py +++ b/python/ray/rllib/utils/actors.py @@ -39,11 +39,8 @@ class TaskPool(object): for worker, obj_id in self.completed(): plasma_id = ray.pyarrow.plasma.ObjectID(obj_id.id()) - if not ray.global_state.use_raylet: - ray.worker.global_worker.plasma_client.fetch([plasma_id]) - else: - (ray.worker.global_worker.local_scheduler_client. - reconstruct_objects([obj_id], True)) + (ray.worker.global_worker.local_scheduler_client. + reconstruct_objects([obj_id], True)) self._fetching.append((worker, obj_id)) remaining = [] diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index 654fba720..fceaabcd2 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -5,7 +5,6 @@ from __future__ import print_function import click import json import logging -import os import subprocess import ray.services as services @@ -20,7 +19,7 @@ logger = logging.getLogger(__name__) def check_no_existing_redis_clients(node_ip_address, redis_client): # The client table prefix must be kept in sync with the file - # "src/common/redis_module/ray_redis_module.cc" where it is defined. + # "src/ray/gcs/redis_module/ray_redis_module.cc" where it is defined. REDIS_CLIENT_TABLE_PREFIX = "CL:" client_keys = redis_client.keys("{}*".format(REDIS_CLIENT_TABLE_PREFIX)) # Filter to clients on the same node and do some basic checking. @@ -167,11 +166,6 @@ def cli(logging_level, logging_format): required=False, type=str, help="the file that contains the autoscaling config") -@click.option( - "--use-raylet", - default=None, - type=bool, - help="use the raylet code path, this defaults to false") @click.option( "--no-redirect-worker-output", is_flag=True, @@ -198,31 +192,15 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards, redis_max_clients, redis_password, redis_shard_ports, object_manager_port, object_store_memory, num_workers, num_cpus, num_gpus, resources, head, no_ui, block, plasma_directory, - huge_pages, autoscaling_config, use_raylet, - no_redirect_worker_output, no_redirect_output, - plasma_store_socket_name, raylet_socket_name, temp_dir): + huge_pages, autoscaling_config, no_redirect_worker_output, + no_redirect_output, plasma_store_socket_name, raylet_socket_name, + temp_dir): # Convert hostnames to numerical IP address. if node_ip_address is not None: node_ip_address = services.address_to_ip(node_ip_address) if redis_address is not None: redis_address = services.address_to_ip(redis_address) - if use_raylet is None: - if os.environ.get("RAY_USE_XRAY") == "0": - # This environment variable is used in our testing setup. - logger.info("Detected environment variable 'RAY_USE_XRAY' with " - "value {}. This turns OFF xray.".format( - os.environ.get("RAY_USE_XRAY"))) - use_raylet = False - else: - use_raylet = True - - if not use_raylet and redis_password is not None: - raise Exception("Setting the 'redis-password' argument is not " - "supported in legacy Ray. To run Ray with " - "password-protected Redis ports, pass " - "the '--use-raylet' flag.") - try: resources = json.loads(resources) except Exception: @@ -290,7 +268,6 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards, plasma_directory=plasma_directory, huge_pages=huge_pages, autoscaling_config=autoscaling_config, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) @@ -369,7 +346,6 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards, resources=resources, plasma_directory=plasma_directory, huge_pages=huge_pages, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) @@ -387,11 +363,7 @@ def start(node_ip_address, redis_address, redis_port, num_redis_shards, @cli.command() def stop(): subprocess.call( - [ - "killall global_scheduler plasma_store_server plasma_manager " - "local_scheduler raylet raylet_monitor" - ], - shell=True) + ["killall plasma_store_server raylet raylet_monitor"], shell=True) # Find the PID of the monitor process and kill it. subprocess.call( diff --git a/python/ray/services.py b/python/ray/services.py index d57bc0429..d0ddd2b32 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -14,32 +14,25 @@ import subprocess import sys import threading import time -from collections import OrderedDict, namedtuple +from collections import OrderedDict import redis import pyarrow # Ray modules import ray.ray_constants -import ray.global_scheduler as global_scheduler -import ray.local_scheduler import ray.plasma from ray.tempfile_services import ( get_ipython_notebook_path, get_logs_dir_path, get_raylet_socket_name, - get_temp_root, new_global_scheduler_log_file, new_local_scheduler_log_file, - new_log_monitor_log_file, new_monitor_log_file, - new_plasma_manager_log_file, new_plasma_store_log_file, - new_raylet_log_file, new_redis_log_file, new_webui_log_file, - new_worker_log_file, set_temp_root) + get_temp_root, new_log_monitor_log_file, new_monitor_log_file, + new_plasma_store_log_file, new_raylet_log_file, new_redis_log_file, + new_webui_log_file, set_temp_root) PROCESS_TYPE_MONITOR = "monitor" PROCESS_TYPE_LOG_MONITOR = "log_monitor" PROCESS_TYPE_WORKER = "worker" PROCESS_TYPE_RAYLET = "raylet" -PROCESS_TYPE_LOCAL_SCHEDULER = "local_scheduler" -PROCESS_TYPE_PLASMA_MANAGER = "plasma_manager" PROCESS_TYPE_PLASMA_STORE = "plasma_store" -PROCESS_TYPE_GLOBAL_SCHEDULER = "global_scheduler" PROCESS_TYPE_REDIS_SERVER = "redis_server" PROCESS_TYPE_WEB_UI = "web_ui" @@ -51,23 +44,20 @@ PROCESS_TYPE_WEB_UI = "web_ui" all_processes = OrderedDict( [(PROCESS_TYPE_MONITOR, []), (PROCESS_TYPE_LOG_MONITOR, []), (PROCESS_TYPE_WORKER, []), (PROCESS_TYPE_RAYLET, []), - (PROCESS_TYPE_LOCAL_SCHEDULER, []), (PROCESS_TYPE_PLASMA_MANAGER, []), - (PROCESS_TYPE_PLASMA_STORE, []), (PROCESS_TYPE_GLOBAL_SCHEDULER, []), - (PROCESS_TYPE_REDIS_SERVER, []), (PROCESS_TYPE_WEB_UI, [])], ) + (PROCESS_TYPE_PLASMA_STORE, []), (PROCESS_TYPE_REDIS_SERVER, []), + (PROCESS_TYPE_WEB_UI, [])], ) # True if processes are run in the valgrind profiler. RUN_RAYLET_PROFILER = False -RUN_LOCAL_SCHEDULER_PROFILER = False -RUN_PLASMA_MANAGER_PROFILER = False RUN_PLASMA_STORE_PROFILER = False # Location of the redis server and module. REDIS_EXECUTABLE = os.path.join( os.path.abspath(os.path.dirname(__file__)), - "core/src/common/thirdparty/redis/src/redis-server") + "core/src/ray/thirdparty/redis/src/redis-server") REDIS_MODULE = os.path.join( os.path.abspath(os.path.dirname(__file__)), - "core/src/common/redis_module/libray_redis_module.so") + "core/src/ray/gcs/redis_module/libray_redis_module.so") # Location of the credis server and modules. # credis will be enabled if the environment variable RAY_USE_NEW_GCS is set. @@ -88,14 +78,6 @@ RAYLET_MONITOR_EXECUTABLE = os.path.join( RAYLET_EXECUTABLE = os.path.join( os.path.abspath(os.path.dirname(__file__)), "core/src/ray/raylet/raylet") -# ObjectStoreAddress tuples contain all information necessary to connect to an -# object store. The fields are: -# - name: The socket name for the object store -# - manager_name: The socket name for the object store manager -# - manager_port: The Internet port that the object store manager listens on -ObjectStoreAddress = namedtuple("ObjectStoreAddress", - ["name", "manager_name", "manager_port"]) - # Logger for this module. It should be configured at the entry point # into the program using Ray. Ray configures it by default automatically # using logging.basicConfig in its entry/init points. @@ -136,10 +118,7 @@ def kill_process(p): if p.poll() is not None: # The process has already terminated. return True - if any([ - RUN_RAYLET_PROFILER, RUN_LOCAL_SCHEDULER_PROFILER, - RUN_PLASMA_MANAGER_PROFILER, RUN_PLASMA_STORE_PROFILER - ]): + if any([RUN_RAYLET_PROFILER, RUN_PLASMA_STORE_PROFILER]): # Give process signal to write profiler data. os.kill(p.pid, signal.SIGINT) # Wait for profiling data to be written. @@ -430,7 +409,6 @@ def start_redis(node_ip_address, redis_shard_ports=None, num_redis_shards=1, redis_max_clients=None, - use_raylet=True, redirect_output=False, redirect_worker_output=False, cleanup=True, @@ -450,7 +428,6 @@ def start_redis(node_ip_address, shard. redis_max_clients: If this is provided, Ray will attempt to configure Redis with this maxclients number. - use_raylet: True if the new raylet code path should be used. redirect_output (bool): True if output should be redirected to a file and false otherwise. redirect_worker_output (bool): True if worker output should be @@ -515,12 +492,6 @@ def start_redis(node_ip_address, port = assigned_port redis_address = address(node_ip_address, port) - redis_client = redis.StrictRedis( - host=node_ip_address, port=port, password=password) - - # Store whether we're using the raylet code path or not. - redis_client.set("UseRaylet", 1 if use_raylet else 0) - # Register the number of Redis shards in the primary shard, so that clients # know how many redis shards to expect under RedisShards. primary_redis_client = redis.StrictRedis( @@ -762,40 +733,6 @@ def start_log_monitor(redis_address, password=redis_password) -def start_global_scheduler(redis_address, - node_ip_address, - stdout_file=None, - stderr_file=None, - cleanup=True, - redis_password=None): - """Start a global scheduler process. - - Args: - redis_address (str): The address of the Redis instance. - node_ip_address: The IP address of the node that this scheduler will - run on. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - cleanup (bool): True if using Ray in local mode. If cleanup is true, - then this process will be killed by services.cleanup() when the - Python process that imported services exits. - redis_password (str): The password of the redis server. - """ - p = global_scheduler.start_global_scheduler( - redis_address, - node_ip_address, - stdout_file=stdout_file, - stderr_file=stderr_file) - if cleanup: - all_processes[PROCESS_TYPE_GLOBAL_SCHEDULER].append(p) - record_log_files_in_redis( - redis_address, - node_ip_address, [stdout_file, stderr_file], - password=redis_password) - - def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True): """Start a UI process. @@ -856,13 +793,11 @@ def start_ui(redis_address, stdout_file=None, stderr_file=None, cleanup=True): return webui_url -def check_and_update_resources(resources, use_raylet): +def check_and_update_resources(resources): """Sanity check a resource dictionary and add sensible defaults. Args: resources: A dictionary mapping resource names to resource quantities. - use_raylet: True if we are using the raylet code path and false - otherwise. Returns: A new resource dictionary. @@ -901,79 +836,13 @@ def check_and_update_resources(resources, use_raylet): and not resource_quantity.is_integer()): raise ValueError("Resource quantities must all be whole numbers.") - if (use_raylet and - resource_quantity > ray.ray_constants.MAX_RESOURCE_QUANTITY): + if resource_quantity > ray.ray_constants.MAX_RESOURCE_QUANTITY: raise ValueError("Resource quantities must be at most {}.".format( ray.ray_constants.MAX_RESOURCE_QUANTITY)) return resources -def start_local_scheduler(redis_address, - node_ip_address, - plasma_store_name, - plasma_manager_name, - worker_path, - plasma_address=None, - stdout_file=None, - stderr_file=None, - cleanup=True, - resources=None, - num_workers=0, - redis_password=None): - """Start a local scheduler process. - - Args: - redis_address (str): The address of the Redis instance. - node_ip_address (str): The IP address of the node that this local - scheduler is running on. - plasma_store_name (str): The name of the plasma store socket to connect - to. - plasma_manager_name (str): The name of the plasma manager socket to - connect to. - worker_path (str): The path of the script to use when the local - scheduler starts up new workers. - stdout_file: A file handle opened for writing to redirect stdout to. If - no redirection should happen, then this should be None. - stderr_file: A file handle opened for writing to redirect stderr to. If - no redirection should happen, then this should be None. - cleanup (bool): True if using Ray in local mode. If cleanup is true, - then this process will be killed by serices.cleanup() when the - Python process that imported services exits. - resources: A dictionary mapping the name of a resource to the available - quantity of that resource. - num_workers (int): The number of workers that the local scheduler - should start. - redis_password (str): The password of the redis server. - - Return: - The name of the local scheduler socket. - """ - resources = check_and_update_resources(resources, False) - - logger.info("Starting local scheduler with the following resources: {}." - .format(resources)) - local_scheduler_name, p = ray.local_scheduler.start_local_scheduler( - plasma_store_name, - plasma_manager_name, - worker_path=worker_path, - node_ip_address=node_ip_address, - redis_address=redis_address, - plasma_address=plasma_address, - use_profiler=RUN_LOCAL_SCHEDULER_PROFILER, - stdout_file=stdout_file, - stderr_file=stderr_file, - static_resources=resources, - num_workers=num_workers) - if cleanup: - all_processes[PROCESS_TYPE_LOCAL_SCHEDULER].append(p) - record_log_files_in_redis( - redis_address, - node_ip_address, [stdout_file, stderr_file], - password=redis_password) - return local_scheduler_name - - def start_raylet(redis_address, node_ip_address, raylet_name, @@ -1017,7 +886,7 @@ def start_raylet(redis_address, if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") - static_resources = check_and_update_resources(resources, True) + static_resources = check_and_update_resources(resources) # Limit the number of workers that can be started in parallel by the # raylet. However, make sure it is at least 1. @@ -1093,13 +962,10 @@ def start_plasma_store(node_ip_address, object_manager_port=None, store_stdout_file=None, store_stderr_file=None, - manager_stdout_file=None, - manager_stderr_file=None, objstore_memory=None, cleanup=True, plasma_directory=None, huge_pages=False, - use_raylet=True, plasma_store_socket_name=None, redis_password=None): """This method starts an object store process. @@ -1114,12 +980,6 @@ def start_plasma_store(node_ip_address, to. If no redirection should happen, then this should be None. store_stderr_file: A file handle opened for writing to redirect stderr to. If no redirection should happen, then this should be None. - manager_stdout_file: A file handle opened for writing to redirect - stdout to. If no redirection should happen, then this should be - None. - manager_stderr_file: A file handle opened for writing to redirect - stderr to. If no redirection should happen, then this should be - None. objstore_memory: The amount of memory (in bytes) to start the object store with. cleanup (bool): True if using Ray in local mode. If cleanup is true, @@ -1129,12 +989,10 @@ def start_plasma_store(node_ip_address, be created. huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. - use_raylet: True if the new raylet code path should be used. redis_password (str): The password of the redis server. Return: - A tuple of the Plasma store socket name, the Plasma manager socket - name, and the plasma manager port. + The Plasma store socket name. """ if objstore_memory is None: # Compute a fraction of the system memory for the Plasma store to use. @@ -1177,32 +1035,6 @@ def start_plasma_store(node_ip_address, plasma_directory=plasma_directory, huge_pages=huge_pages, socket_name=plasma_store_socket_name) - # Start the plasma manager. - if not use_raylet: - if object_manager_port is not None: - (plasma_manager_name, p2, - plasma_manager_port) = ray.plasma.start_plasma_manager( - plasma_store_name, - redis_address, - plasma_manager_port=object_manager_port, - node_ip_address=node_ip_address, - num_retries=1, - run_profiler=RUN_PLASMA_MANAGER_PROFILER, - stdout_file=manager_stdout_file, - stderr_file=manager_stderr_file) - assert plasma_manager_port == object_manager_port - else: - (plasma_manager_name, p2, - plasma_manager_port) = ray.plasma.start_plasma_manager( - plasma_store_name, - redis_address, - node_ip_address=node_ip_address, - run_profiler=RUN_PLASMA_MANAGER_PROFILER, - stdout_file=manager_stdout_file, - stderr_file=manager_stderr_file) - else: - plasma_manager_port = None - plasma_manager_name = None if cleanup: all_processes[PROCESS_TYPE_PLASMA_STORE].append(p1) @@ -1210,19 +1042,12 @@ def start_plasma_store(node_ip_address, redis_address, node_ip_address, [store_stdout_file, store_stderr_file], password=redis_password) - if not use_raylet: - if cleanup: - all_processes[PROCESS_TYPE_PLASMA_MANAGER].append(p2) - record_log_files_in_redis(redis_address, node_ip_address, - [manager_stdout_file, manager_stderr_file]) - return ObjectStoreAddress(plasma_store_name, plasma_manager_name, - plasma_manager_port) + return plasma_store_name def start_worker(node_ip_address, object_store_name, - object_store_manager_name, local_scheduler_name, redis_address, worker_path, @@ -1235,7 +1060,6 @@ def start_worker(node_ip_address, node_ip_address (str): The IP address of the node that this worker is running on. object_store_name (str): The name of the object store. - object_store_manager_name (str): The name of the object store manager. local_scheduler_name (str): The name of the local scheduler. redis_address (str): The address that the Redis server is listening on. worker_path (str): The path of the source code which the worker process @@ -1253,7 +1077,6 @@ def start_worker(node_ip_address, sys.executable, "-u", worker_path, "--node-ip-address=" + node_ip_address, "--object-store-name=" + object_store_name, - "--object-store-manager-name=" + object_store_manager_name, "--local-scheduler-name=" + local_scheduler_name, "--redis-address=" + str(redis_address), "--temp-dir=" + get_temp_root() @@ -1349,7 +1172,6 @@ def start_ray_processes(address_info=None, cleanup=True, redirect_worker_output=False, redirect_output=False, - include_global_scheduler=False, include_log_monitor=False, include_webui=False, start_workers_from_local_scheduler=True, @@ -1357,7 +1179,6 @@ def start_ray_processes(address_info=None, plasma_directory=None, huge_pages=False, autoscaling_config=None, - use_raylet=True, plasma_store_socket_name=None, raylet_socket_name=None, temp_dir=None): @@ -1398,8 +1219,6 @@ def start_ray_processes(address_info=None, processes should be redirected to files. redirect_output (bool): True if stdout and stderr for non-worker processes should be redirected to files and false otherwise. - include_global_scheduler (bool): If include_global_scheduler is True, - then start a global scheduler process. include_log_monitor (bool): If True, then start a log monitor to monitor the log files for all processes on this node and push their contents to Redis. @@ -1415,7 +1234,6 @@ def start_ray_processes(address_info=None, huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. autoscaling_config: path to autoscaling config file. - use_raylet: True if the new raylet code path should be used. plasma_store_socket_name (str): If provided, it will specify the socket name used by the plasma store. raylet_socket_name (str): If provided, it will specify the socket path @@ -1469,7 +1287,6 @@ def start_ray_processes(address_info=None, redis_shard_ports=redis_shard_ports, num_redis_shards=num_redis_shards, redis_max_clients=redis_max_clients, - use_raylet=use_raylet, redirect_output=True, redirect_worker_output=redirect_worker_output, cleanup=cleanup, @@ -1488,13 +1305,12 @@ def start_ray_processes(address_info=None, cleanup=cleanup, autoscaling_config=autoscaling_config, redis_password=redis_password) - if use_raylet: - start_raylet_monitor( - redis_address, - stdout_file=monitor_stdout_file, - stderr_file=monitor_stderr_file, - cleanup=cleanup, - redis_password=redis_password) + start_raylet_monitor( + redis_address, + stdout_file=monitor_stdout_file, + stderr_file=monitor_stderr_file, + cleanup=cleanup, + redis_password=redis_password) if redis_shards == []: # Get redis shards from primary redis instance. redis_ip_address, redis_port = redis_address.split(":") @@ -1516,25 +1332,10 @@ def start_ray_processes(address_info=None, cleanup=cleanup, redis_password=redis_password) - # Start the global scheduler, if necessary. - if include_global_scheduler and not use_raylet: - global_scheduler_stdout_file, global_scheduler_stderr_file = ( - new_global_scheduler_log_file(redirect_output)) - start_global_scheduler( - redis_address, - node_ip_address, - stdout_file=global_scheduler_stdout_file, - stderr_file=global_scheduler_stderr_file, - cleanup=cleanup, - redis_password=redis_password) - # Initialize with existing services. if "object_store_addresses" not in address_info: address_info["object_store_addresses"] = [] object_store_addresses = address_info["object_store_addresses"] - if "local_scheduler_socket_names" not in address_info: - address_info["local_scheduler_socket_names"] = [] - local_scheduler_socket_names = address_info["local_scheduler_socket_names"] if "raylet_socket_names" not in address_info: address_info["raylet_socket_names"] = [] raylet_socket_names = address_info["raylet_socket_names"] @@ -1552,114 +1353,37 @@ def start_ray_processes(address_info=None, plasma_store_stdout_file, plasma_store_stderr_file = ( new_plasma_store_log_file(i, redirect_output)) - # If we use raylet, plasma manager won't be started and we don't need - # to create temp files for them. - plasma_manager_stdout_file, plasma_manager_stderr_file = ( - new_plasma_manager_log_file(i, redirect_output and not use_raylet)) - object_store_address = start_plasma_store( node_ip_address, redis_address, - object_manager_port=object_manager_ports[i], store_stdout_file=plasma_store_stdout_file, store_stderr_file=plasma_store_stderr_file, - manager_stdout_file=plasma_manager_stdout_file, - manager_stderr_file=plasma_manager_stderr_file, objstore_memory=object_store_memory, cleanup=cleanup, plasma_directory=plasma_directory, huge_pages=huge_pages, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, redis_password=redis_password) object_store_addresses.append(object_store_address) time.sleep(0.1) - if not use_raylet: - # Start any local schedulers that do not yet exist. - for i in range( - len(local_scheduler_socket_names), num_local_schedulers): - # Connect the local scheduler to the object store at the same - # index. - object_store_address = object_store_addresses[i] - plasma_address = "{}:{}".format(node_ip_address, - object_store_address.manager_port) - # Determine how many workers this local scheduler should start. - if start_workers_from_local_scheduler: - num_local_scheduler_workers = workers_per_local_scheduler[i] - workers_per_local_scheduler[i] = 0 - else: - # If we're starting the workers from Python, the local - # scheduler should not start any workers. - num_local_scheduler_workers = 0 - # Start the local scheduler. Note that if we do not wish to - # redirect the worker output, then we cannot redirect the local - # scheduler output. - local_scheduler_stdout_file, local_scheduler_stderr_file = ( - new_local_scheduler_log_file( - i, redirect_output=redirect_worker_output)) - local_scheduler_name = start_local_scheduler( + # Start any raylets that do not exist yet. + for i in range(len(raylet_socket_names), num_local_schedulers): + raylet_stdout_file, raylet_stderr_file = new_raylet_log_file( + i, redirect_output=redirect_worker_output) + address_info["raylet_socket_names"].append( + start_raylet( redis_address, node_ip_address, - object_store_address.name, - object_store_address.manager_name, + raylet_socket_name or get_raylet_socket_name(), + object_store_addresses[i], worker_path, - plasma_address=plasma_address, - stdout_file=local_scheduler_stdout_file, - stderr_file=local_scheduler_stderr_file, - cleanup=cleanup, resources=resources[i], - num_workers=num_local_scheduler_workers, - redis_password=redis_password) - local_scheduler_socket_names.append(local_scheduler_name) - - # Make sure that we have exactly num_local_schedulers instances of - # object stores and local schedulers. - assert len(object_store_addresses) == num_local_schedulers - assert len(local_scheduler_socket_names) == num_local_schedulers - - else: - # Start any raylets that do not exist yet. - for i in range(len(raylet_socket_names), num_local_schedulers): - raylet_stdout_file, raylet_stderr_file = new_raylet_log_file( - i, redirect_output=redirect_worker_output) - address_info["raylet_socket_names"].append( - start_raylet( - redis_address, - node_ip_address, - raylet_socket_name or get_raylet_socket_name(), - object_store_addresses[i].name, - worker_path, - resources=resources[i], - num_workers=workers_per_local_scheduler[i], - stdout_file=raylet_stdout_file, - stderr_file=raylet_stderr_file, - cleanup=cleanup, - redis_password=redis_password)) - - if not use_raylet: - # Start any workers that the local scheduler has not already started. - for i, num_local_scheduler_workers in enumerate( - workers_per_local_scheduler): - object_store_address = object_store_addresses[i] - local_scheduler_name = local_scheduler_socket_names[i] - for j in range(num_local_scheduler_workers): - worker_stdout_file, worker_stderr_file = new_worker_log_file( - i, j, redirect_output) - start_worker( - node_ip_address, - object_store_address.name, - object_store_address.manager_name, - local_scheduler_name, - redis_address, - worker_path, - stdout_file=worker_stdout_file, - stderr_file=worker_stderr_file, - cleanup=cleanup) - workers_per_local_scheduler[i] -= 1 - - # Make sure that we've started all the workers. - assert (sum(workers_per_local_scheduler) == 0) + num_workers=workers_per_local_scheduler[i], + stdout_file=raylet_stdout_file, + stderr_file=raylet_stderr_file, + cleanup=cleanup, + redis_password=redis_password)) # Try to start the web UI. if include_webui: @@ -1689,7 +1413,6 @@ def start_ray_node(node_ip_address, resources=None, plasma_directory=None, huge_pages=False, - use_raylet=True, plasma_store_socket_name=None, raylet_socket_name=None, temp_dir=None): @@ -1727,7 +1450,6 @@ def start_ray_node(node_ip_address, be created. huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. - use_raylet: True if the new raylet code path should be used. plasma_store_socket_name (str): If provided, it will specify the socket name used by the plasma store. raylet_socket_name (str): If provided, it will specify the socket path @@ -1758,7 +1480,6 @@ def start_ray_node(node_ip_address, resources=resources, plasma_directory=plasma_directory, huge_pages=huge_pages, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) @@ -1784,7 +1505,6 @@ def start_ray_head(address_info=None, plasma_directory=None, huge_pages=False, autoscaling_config=None, - use_raylet=True, plasma_store_socket_name=None, raylet_socket_name=None, temp_dir=None): @@ -1836,7 +1556,6 @@ def start_ray_head(address_info=None, huge_pages: Boolean flag indicating whether to start the Object Store with hugetlbfs support. Requires plasma_directory. autoscaling_config: path to autoscaling config file. - use_raylet: True if the new raylet code path should be used. plasma_store_socket_name (str): If provided, it will specify the socket name used by the plasma store. raylet_socket_name (str): If provided, it will specify the socket path @@ -1861,7 +1580,6 @@ def start_ray_head(address_info=None, cleanup=cleanup, redirect_worker_output=redirect_worker_output, redirect_output=redirect_output, - include_global_scheduler=True, include_log_monitor=True, include_webui=include_webui, start_workers_from_local_scheduler=start_workers_from_local_scheduler, @@ -1872,7 +1590,6 @@ def start_ray_head(address_info=None, plasma_directory=plasma_directory, huge_pages=huge_pages, autoscaling_config=autoscaling_config, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) diff --git a/python/ray/tempfile_services.py b/python/ray/tempfile_services.py index 76ec7c1d7..bf8b6c219 100644 --- a/python/ray/tempfile_services.py +++ b/python/ray/tempfile_services.py @@ -117,27 +117,6 @@ def get_object_store_socket_name(): return make_inc_temp(prefix="plasma_store", directory_name=sockets_dir) -def get_plasma_manager_socket_name(): - """Get a socket name for plasma manager.""" - sockets_dir = get_sockets_dir_path() - return make_inc_temp(prefix="plasma_manager", directory_name=sockets_dir) - - -def get_local_scheduler_socket_name(suffix=""): - """Get a socket name for local scheduler. - - This function could be unsafe. The socket name may - refer to a file that did not exist at some point, but by the time - you get around to creating it, someone else may have beaten you to - the punch. - """ - sockets_dir = get_sockets_dir_path() - raylet_socket_name = make_inc_temp( - prefix="scheduler", directory_name=sockets_dir, suffix=suffix) - - return raylet_socket_name - - def get_ipython_notebook_path(port): """Get a new ipython notebook path""" @@ -211,17 +190,6 @@ def new_raylet_log_file(local_scheduler_index, redirect_output): return raylet_stdout_file, raylet_stderr_file -def new_local_scheduler_log_file(local_scheduler_index, redirect_output): - """Create new logging files for local scheduler. - - It is only used in non-raylet versions. - """ - local_scheduler_stdout_file, local_scheduler_stderr_file = (new_log_files( - "local_scheduler_{}".format(local_scheduler_index), - redirect_output=redirect_output)) - return local_scheduler_stdout_file, local_scheduler_stderr_file - - def new_webui_log_file(): """Create new logging files for web ui.""" ui_stdout_file, ui_stderr_file = new_log_files( @@ -229,17 +197,6 @@ def new_webui_log_file(): return ui_stdout_file, ui_stderr_file -def new_worker_log_file(local_scheduler_index, worker_index, redirect_output): - """Create new logging files for workers with local scheduler index. - - It is only used in non-raylet versions. - """ - worker_stdout_file, worker_stderr_file = new_log_files( - "worker_{}_{}".format(local_scheduler_index, worker_index), - redirect_output) - return worker_stdout_file, worker_stderr_file - - def new_worker_redirected_log_file(worker_id): """Create new logging files for workers to redirect its output.""" worker_stdout_file, worker_stderr_file = (new_log_files( @@ -254,16 +211,6 @@ def new_log_monitor_log_file(): return log_monitor_stdout_file, log_monitor_stderr_file -def new_global_scheduler_log_file(redirect_output): - """Create new logging files for the new global scheduler. - - It is only used in non-raylet versions. - """ - global_scheduler_stdout_file, global_scheduler_stderr_file = ( - new_log_files("global_scheduler", redirect_output)) - return global_scheduler_stdout_file, global_scheduler_stderr_file - - def new_plasma_store_log_file(local_scheduler_index, redirect_output): """Create new logging files for the plasma store.""" plasma_store_stdout_file, plasma_store_stderr_file = new_log_files( @@ -271,13 +218,6 @@ def new_plasma_store_log_file(local_scheduler_index, redirect_output): return plasma_store_stdout_file, plasma_store_stderr_file -def new_plasma_manager_log_file(local_scheduler_index, redirect_output): - """Create new logging files for the plasma manager.""" - plasma_manager_stdout_file, plasma_manager_stderr_file = new_log_files( - "plasma_manager_{}".format(local_scheduler_index), redirect_output) - return plasma_manager_stdout_file, plasma_manager_stderr_file - - def new_monitor_log_file(redirect_output): """Create new logging files for the monitor.""" monitor_stdout_file, monitor_stderr_file = new_log_files( diff --git a/python/ray/test/cluster_utils.py b/python/ray/test/cluster_utils.py index 7e7a82d67..c4cf2b801 100644 --- a/python/ray/test/cluster_utils.py +++ b/python/ray/test/cluster_utils.py @@ -44,7 +44,6 @@ class Cluster(object): All nodes are by default started with the following settings: cleanup=True, - use_raylet=True, resources={"CPU": 1}, object_store_memory=100 * (2**20) # 100 MB @@ -55,12 +54,13 @@ class Cluster(object): Returns: Node object of the added Ray node. """ - node_kwargs = dict( - cleanup=True, - use_raylet=True, - resources={"CPU": 1}, - object_store_memory=100 * (2**20) # 100 MB - ) + node_kwargs = { + "cleanup": True, + "resources": { + "CPU": 1 + }, + "object_store_memory": 100 * (2**20) # 100 MB + } node_kwargs.update(override_kwargs) if self.head_node is None: @@ -179,7 +179,9 @@ class Node(object): for process_name, process_list in self.process_dict.items(): logger.info("Killing all {}(s)".format(process_name)) for process in process_list: - process.kill() + # Kill the process if it is still alive. + if process.poll() is None: + process.kill() for process_name, process_list in self.process_dict.items(): logger.info("Waiting all {}(s)".format(process_name)) diff --git a/python/ray/test/test_ray_init.py b/python/ray/test/test_ray_init.py index a64dd4a94..3b2beaba1 100644 --- a/python/ray/test/test_ray_init.py +++ b/python/ray/test/test_ray_init.py @@ -28,9 +28,6 @@ class TestRedisPassword(object): @pytest.mark.skipif( os.environ.get("RAY_USE_NEW_GCS") == "on", reason="New GCS API doesn't support Redis authentication yet.") - @pytest.mark.skipif( - os.environ.get("RAY_USE_XRAY") == "0", - reason="Redis authentication is not supported in legacy Ray.") def test_redis_password(self, password, shutdown_only): # Workaround for https://github.com/ray-project/ray/issues/3045 @ray.remote diff --git a/python/ray/test/test_utils.py b/python/ray/test/test_utils.py index 6e18cd439..a3614650e 100644 --- a/python/ray/test/test_utils.py +++ b/python/ray/test/test_utils.py @@ -35,22 +35,11 @@ def _wait_for_nodes_to_join(num_nodes, timeout=20): client_table = ray.global_state.client_table() num_ready_nodes = len(client_table) if num_ready_nodes == num_nodes: - ready = True # Check that for each node, a local scheduler and a plasma manager # are present. - if ray.global_state.use_raylet: - # In raylet mode, this is a list of map. - # The GCS info will appear as a whole instead of part by part. - return - else: - for ip_address, clients in client_table.items(): - client_types = [client["ClientType"] for client in clients] - if "local_scheduler" not in client_types: - ready = False - if "plasma_manager" not in client_types: - ready = False - if ready: - return + # In raylet mode, this is a list of map. + # The GCS info will appear as a whole instead of part by part. + return if num_ready_nodes > num_nodes: # Too many nodes have joined. Something must be wrong. raise Exception("{} nodes have joined the cluster, but we were " diff --git a/python/ray/tune/ray_trial_executor.py b/python/ray/tune/ray_trial_executor.py index acbebb38b..4a216a60d 100644 --- a/python/ray/tune/ray_trial_executor.py +++ b/python/ray/tune/ray_trial_executor.py @@ -213,20 +213,9 @@ class RayTrialExecutor(TrialExecutor): assert self._committed_resources.gpu >= 0 def _update_avail_resources(self): - if ray.worker.global_worker.use_raylet: - # TODO(rliaw): Remove once raylet flag is swapped - resources = ray.global_state.cluster_resources() - num_cpus = resources["CPU"] - num_gpus = resources["GPU"] - else: - clients = ray.global_state.client_table() - local_schedulers = [ - entry for client in clients.values() for entry in client - if (entry['ClientType'] == 'local_scheduler' - and not entry['Deleted']) - ] - num_cpus = sum(ls['CPU'] for ls in local_schedulers) - num_gpus = sum(ls.get('GPU', 0) for ls in local_schedulers) + resources = ray.global_state.cluster_resources() + num_cpus = resources["CPU"] + num_gpus = resources["GPU"] self._avail_resources = Resources(int(num_cpus), int(num_gpus)) self._resources_initialized = True diff --git a/python/ray/tune/test/trial_runner_test.py b/python/ray/tune/test/trial_runner_test.py index 450e96136..3c9ae43e6 100644 --- a/python/ray/tune/test/trial_runner_test.py +++ b/python/ray/tune/test/trial_runner_test.py @@ -107,7 +107,7 @@ class TrainableFunctionApiTest(unittest.TestCase): return Resources(cpu=config["cpu"], gpu=config["gpu"]) def _train(self): - return dict(timesteps_this_iter=1, done=True) + return {"timesteps_this_iter": 1, "done": True} register_trainable("B", B) @@ -440,7 +440,7 @@ class TrainableFunctionApiTest(unittest.TestCase): self.state = {"hi": 1} def _train(self): - return dict(timesteps_this_iter=1, done=True) + return {"timesteps_this_iter": 1, "done": True} def _save(self, path): return self.state @@ -471,7 +471,7 @@ class TrainableFunctionApiTest(unittest.TestCase): def _train(self): self.state["iter"] += 1 - return dict(timesteps_this_iter=1, done=True) + return {"timesteps_this_iter": 1, "done": True} def _save(self, path): return self.state @@ -604,7 +604,7 @@ class RunExperimentTest(unittest.TestCase): class B(Trainable): def _train(self): - return dict(timesteps_this_iter=1, done=True) + return {"timesteps_this_iter": 1, "done": True} register_trainable("f1", train) trials = run_experiments({ @@ -624,7 +624,7 @@ class RunExperimentTest(unittest.TestCase): def testCheckpointAtEnd(self): class train(Trainable): def _train(self): - return dict(timesteps_this_iter=1, done=True) + return {"timesteps_this_iter": 1, "done": True} def _save(self, path): return path @@ -887,7 +887,7 @@ class TrialRunnerTest(unittest.TestCase): self.assertEqual(trials[1].status, Trial.PENDING) def testFractionalGpus(self): - ray.init(num_cpus=4, num_gpus=1, use_raylet=True) + ray.init(num_cpus=4, num_gpus=1) runner = TrialRunner(BasicVariantGenerator()) kwargs = { "resources": Resources(cpu=1, gpu=0.5), diff --git a/python/ray/tune/util.py b/python/ray/tune/util.py index 691d25adb..9c047fd80 100644 --- a/python/ray/tune/util.py +++ b/python/ray/tune/util.py @@ -28,7 +28,7 @@ def pin_in_object_store(obj): def get_pinned_object(pinned_id): """Retrieve a pinned object from the object store.""" - from ray.local_scheduler import ObjectID + from ray.raylet import ObjectID return _from_pinnable( ray.get( diff --git a/python/ray/utils.py b/python/ray/utils.py index 55f85c8ac..83e2ae2f7 100644 --- a/python/ray/utils.py +++ b/python/ray/utils.py @@ -15,11 +15,9 @@ import time import uuid import ray.gcs_utils -import ray.local_scheduler +import ray.raylet import ray.ray_constants as ray_constants -ERROR_KEY_PREFIX = b"Error:" - def _random_string(): id_hash = hashlib.sha1() @@ -70,22 +68,12 @@ def push_error_to_driver(worker, """ if driver_id is None: driver_id = ray_constants.NIL_JOB_ID.id() - error_key = ERROR_KEY_PREFIX + driver_id + b":" + _random_string() data = {} if data is None else data - if not worker.use_raylet: - worker.redis_client.hmset(error_key, { - "type": error_type, - "message": message, - "data": data - }) - worker.redis_client.rpush("ErrorKeys", error_key) - else: - worker.local_scheduler_client.push_error( - ray.ObjectID(driver_id), error_type, message, time.time()) + worker.local_scheduler_client.push_error( + ray.ObjectID(driver_id), error_type, message, time.time()) def push_error_to_driver_through_redis(redis_client, - use_raylet, error_type, message, driver_id=None, @@ -99,8 +87,6 @@ def push_error_to_driver_through_redis(redis_client, Args: redis_client: The redis client to use. - use_raylet: True if we are using the Raylet code path and false - otherwise. error_type (str): The type of the error. message (str): The message that will be printed in the background on the driver. @@ -111,23 +97,14 @@ def push_error_to_driver_through_redis(redis_client, """ if driver_id is None: driver_id = ray_constants.NIL_JOB_ID.id() - error_key = ERROR_KEY_PREFIX + driver_id + b":" + _random_string() data = {} if data is None else data - if not use_raylet: - redis_client.hmset(error_key, { - "type": error_type, - "message": message, - "data": data - }) - redis_client.rpush("ErrorKeys", error_key) - else: - # Do everything in Python and through the Python Redis client instead - # of through the raylet. - error_data = ray.gcs_utils.construct_error_message( - driver_id, error_type, message, time.time()) - redis_client.execute_command( - "RAY.TABLE_APPEND", ray.gcs_utils.TablePrefix.ERROR_INFO, - ray.gcs_utils.TablePubsub.ERROR_INFO, driver_id, error_data) + # Do everything in Python and through the Python Redis client instead + # of through the raylet. + error_data = ray.gcs_utils.construct_error_message(driver_id, error_type, + message, time.time()) + redis_client.execute_command( + "RAY.TABLE_APPEND", ray.gcs_utils.TablePrefix.ERROR_INFO, + ray.gcs_utils.TablePubsub.ERROR_INFO, driver_id, error_data) def is_cython(obj): diff --git a/python/ray/worker.py b/python/ray/worker.py index 266b995f6..0b042dc11 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -27,14 +27,13 @@ import ray.serialization as serialization import ray.services as services import ray.signature import ray.tempfile_services as tempfile_services -import ray.local_scheduler +import ray.raylet import ray.plasma import ray.ray_constants as ray_constants from ray import import_thread from ray import profiling from ray.function_manager import FunctionActorManager from ray.utils import ( - binary_to_hex, check_oversized_pickle, is_cython, random_string, @@ -56,14 +55,6 @@ NIL_ACTOR_ID = NIL_ID NIL_ACTOR_HANDLE_ID = NIL_ID NIL_CLIENT_ID = ray_constants.ID_SIZE * b"\xff" -# This must be kept in sync with the `error_types` array in -# common/state/error_table.h. -OBJECT_HASH_MISMATCH_ERROR_TYPE = b"object_hash_mismatch" -PUT_RECONSTRUCTION_ERROR_TYPE = b"put_reconstruction" - -# This must be kept in sync with the `scheduling_state` enum in common/task.h. -TASK_STATUS_RUNNING = 8 - # Default resource requirements for actors when no resource requirements are # specified. DEFAULT_ACTOR_METHOD_CPUS_SIMPLE_CASE = 1 @@ -461,13 +452,9 @@ class Worker(object): ] for i in range(0, len(object_ids), ray._config.worker_fetch_request_size()): - if not self.use_raylet: - self.plasma_client.fetch(plain_object_ids[i:( - i + ray._config.worker_fetch_request_size())]) - else: - self.local_scheduler_client.reconstruct_objects( - object_ids[i:( - i + ray._config.worker_fetch_request_size())], True) + self.local_scheduler_client.reconstruct_objects( + object_ids[i:(i + ray._config.worker_fetch_request_size())], + True) # Get the objects. We initially try to get the objects immediately. final_results = self.retrieve_and_deserialize(plain_object_ids, 0) @@ -497,25 +484,9 @@ class Worker(object): ray._config.worker_fetch_request_size()) for i in range(0, len(object_ids_to_fetch), fetch_request_size): - if not self.use_raylet: - for unready_id in ray_object_ids_to_fetch[i:( - i + fetch_request_size)]: - (self.local_scheduler_client. - reconstruct_objects([unready_id], False)) - # Do another fetch for objects that aren't - # available locally yet, in case they were evicted - # since the last fetch. We divide the fetch into - # smaller fetches so as to not block the manager - # for a prolonged period of time in a single call. - # This is only necessary for legacy ray since - # reconstruction and fetch are implemented by - # different processes. - self.plasma_client.fetch(object_ids_to_fetch[i:( - i + fetch_request_size)]) - else: - self.local_scheduler_client.reconstruct_objects( - ray_object_ids_to_fetch[i:( - i + fetch_request_size)], False) + self.local_scheduler_client.reconstruct_objects( + ray_object_ids_to_fetch[i:( + i + fetch_request_size)], False) results = self.retrieve_and_deserialize( object_ids_to_fetch, max([ @@ -608,7 +579,7 @@ class Worker(object): for arg in args: if isinstance(arg, ray.ObjectID): args_for_local_scheduler.append(arg) - elif ray.local_scheduler.check_simple_value(arg): + elif ray.raylet.check_simple_value(arg): args_for_local_scheduler.append(arg) else: args_for_local_scheduler.append(put(arg)) @@ -641,14 +612,13 @@ class Worker(object): task_index = self.task_index self.task_index += 1 # Submit the task to local scheduler. - task = ray.local_scheduler.Task( + task = ray.raylet.Task( driver_id, ray.ObjectID( function_id.id()), args_for_local_scheduler, num_return_vals, self.current_task_id, task_index, actor_creation_id, actor_creation_dummy_object_id, actor_id, - actor_handle_id, actor_counter, is_actor_checkpoint_method, - execution_dependencies, resources, placement_resources, - self.use_raylet) + actor_handle_id, actor_counter, execution_dependencies, + resources, placement_resources) self.local_scheduler_client.submit(task) return task.returns() @@ -925,26 +895,13 @@ class Worker(object): # good to know where the system is hanging. with self.lock: function_name = execution_info.function_name - if not self.use_raylet: - extra_data = { - "function_name": function_name, - "task_id": task.task_id().hex(), - "worker_id": binary_to_hex(self.worker_id) - } - else: - extra_data = { - "name": function_name, - "task_id": task.task_id().hex() - } + extra_data = { + "name": function_name, + "task_id": task.task_id().hex() + } with profiling.profile("task", extra_data=extra_data, worker=self): self._process_task(task, execution_info) - # In the non-raylet code path, push all of the log events to the global - # state store. In the raylet code path, this is done periodically in a - # background thread. - if not self.use_raylet: - self.profiler.flush_profile_data() - # Increase the task execution counter. self.function_actor_manager.increase_task_counter( driver_id, function_id.id()) @@ -998,13 +955,10 @@ def get_gpu_ids(): raise Exception("ray.get_gpu_ids() currently does not work in PYTHON " "MODE.") - if not global_worker.use_raylet: - assigned_ids = global_worker.local_scheduler_client.gpu_ids() - else: - all_resource_ids = global_worker.local_scheduler_client.resource_ids() - assigned_ids = [ - resource_id for resource_id, _ in all_resource_ids.get("GPU", []) - ] + all_resource_ids = global_worker.local_scheduler_client.resource_ids() + assigned_ids = [ + resource_id for resource_id, _ in all_resource_ids.get("GPU", []) + ] # If the user had already set CUDA_VISIBLE_DEVICES, then respect that (in # the sense that only GPU IDs that appear in CUDA_VISIBLE_DEVICES should be # returned). @@ -1019,17 +973,11 @@ def get_gpu_ids(): def get_resource_ids(): """Get the IDs of the resources that are available to the worker. - This function is only supported in the raylet code path. - Returns: A dictionary mapping the name of a resource to a list of pairs, where each pair consists of the ID of a resource and the fraction of that resource reserved for this worker. """ - if not global_worker.use_raylet: - raise Exception("ray.get_resource_ids() is only supported in the " - "raylet code path.") - if _mode() == LOCAL_MODE: raise Exception( "ray.get_resource_ids() currently does not work in PYTHON " @@ -1112,22 +1060,8 @@ def error_applies_to_driver(error_key, worker=global_worker): def error_info(worker=global_worker): """Return information about failed tasks.""" worker.check_connected() - if worker.use_raylet: - return (global_state.error_messages(job_id=worker.task_driver_id) + - global_state.error_messages(job_id=ray_constants.NIL_JOB_ID)) - error_keys = worker.redis_client.lrange("ErrorKeys", 0, -1) - errors = [] - for error_key in error_keys: - if error_applies_to_driver(error_key, worker=worker): - error_contents = worker.redis_client.hgetall(error_key) - error_contents = { - "type": ray.utils.decode(error_contents[b"type"]), - "message": ray.utils.decode(error_contents[b"message"]), - "data": ray.utils.decode(error_contents[b"data"]) - } - errors.append(error_contents) - - return errors + return (global_state.error_messages(job_id=worker.task_driver_id) + + global_state.error_messages(job_id=ray_constants.NIL_JOB_ID)) def _initialize_serialization(driver_id, worker=global_worker): @@ -1223,7 +1157,6 @@ def _initialize_serialization(driver_id, worker=global_worker): def get_address_info_from_redis_helper(redis_address, node_ip_address, - use_raylet=True, redis_password=None): redis_ip_address, redis_port = redis_address.split(":") # For this command to work, some other client (on the same machine as @@ -1231,118 +1164,50 @@ def get_address_info_from_redis_helper(redis_address, redis_client = redis.StrictRedis( host=redis_ip_address, port=int(redis_port), password=redis_password) - if not use_raylet: - # The client table prefix must be kept in sync with the file - # "src/common/redis_module/ray_redis_module.cc" where it is defined. - client_keys = redis_client.keys("{}*".format( - ray.gcs_utils.DB_CLIENT_PREFIX)) - # Filter to live clients on the same node and do some basic checking. - plasma_managers = [] - local_schedulers = [] - for key in client_keys: - info = redis_client.hgetall(key) - - # Ignore clients that were deleted. - deleted = info[b"deleted"] - deleted = bool(int(deleted)) - if deleted: - continue - - assert b"ray_client_id" in info - assert b"node_ip_address" in info - assert b"client_type" in info - client_node_ip_address = ray.utils.decode(info[b"node_ip_address"]) - if (client_node_ip_address == node_ip_address or - (client_node_ip_address == "127.0.0.1" - and redis_ip_address == ray.services.get_node_ip_address())): - if ray.utils.decode(info[b"client_type"]) == "plasma_manager": - plasma_managers.append(info) - elif (ray.utils.decode( - info[b"client_type"]) == "local_scheduler"): - local_schedulers.append(info) - # Make sure that we got at least one plasma manager and local - # scheduler. - assert len(plasma_managers) >= 1 - assert len(local_schedulers) >= 1 - # Build the address information. - object_store_addresses = [] - for manager in plasma_managers: - address = ray.utils.decode(manager[b"manager_address"]) - port = services.get_port(address) - object_store_addresses.append( - services.ObjectStoreAddress( - name=ray.utils.decode(manager[b"store_socket_name"]), - manager_name=ray.utils.decode( - manager[b"manager_socket_name"]), - manager_port=port)) - scheduler_names = [ - ray.utils.decode(scheduler[b"local_scheduler_socket_name"]) - for scheduler in local_schedulers - ] - client_info = { - "node_ip_address": node_ip_address, - "redis_address": redis_address, - "object_store_addresses": object_store_addresses, - "local_scheduler_socket_names": scheduler_names, - # Web UI should be running. - "webui_url": _webui_url_helper(redis_client) - } - return client_info - - # Handle the raylet case. - else: - # In the raylet code path, all client data is stored in a zset at the - # key for the nil client. - client_key = b"CLIENT" + NIL_CLIENT_ID - clients = redis_client.zrange(client_key, 0, -1) - raylets = [] - for client_message in clients: - client = ray.gcs_utils.ClientTableData.GetRootAsClientTableData( - client_message, 0) - client_node_ip_address = ray.utils.decode( - client.NodeManagerAddress()) - if (client_node_ip_address == node_ip_address or - (client_node_ip_address == "127.0.0.1" - and redis_ip_address == ray.services.get_node_ip_address())): - raylets.append(client) - # Make sure that at least one raylet has started locally. - # This handles a race condition where Redis has started but - # the raylet has not connected. - if len(raylets) == 0: - raise Exception( - "Redis has started but no raylets have registered yet.") - object_store_addresses = [ - services.ObjectStoreAddress( - name=ray.utils.decode(raylet.ObjectStoreSocketName()), - manager_name=None, - manager_port=None) for raylet in raylets - ] - raylet_socket_names = [ - ray.utils.decode(raylet.RayletSocketName()) for raylet in raylets - ] - return { - "node_ip_address": node_ip_address, - "redis_address": redis_address, - "object_store_addresses": object_store_addresses, - "raylet_socket_names": raylet_socket_names, - # Web UI should be running. - "webui_url": _webui_url_helper(redis_client) - } + # In the raylet code path, all client data is stored in a zset at the + # key for the nil client. + client_key = b"CLIENT" + NIL_CLIENT_ID + clients = redis_client.zrange(client_key, 0, -1) + raylets = [] + for client_message in clients: + client = ray.gcs_utils.ClientTableData.GetRootAsClientTableData( + client_message, 0) + client_node_ip_address = ray.utils.decode(client.NodeManagerAddress()) + if (client_node_ip_address == node_ip_address or + (client_node_ip_address == "127.0.0.1" + and redis_ip_address == ray.services.get_node_ip_address())): + raylets.append(client) + # Make sure that at least one raylet has started locally. + # This handles a race condition where Redis has started but + # the raylet has not connected. + if len(raylets) == 0: + raise Exception( + "Redis has started but no raylets have registered yet.") + object_store_addresses = [ + ray.utils.decode(raylet.ObjectStoreSocketName()) for raylet in raylets + ] + raylet_socket_names = [ + ray.utils.decode(raylet.RayletSocketName()) for raylet in raylets + ] + return { + "node_ip_address": node_ip_address, + "redis_address": redis_address, + "object_store_addresses": object_store_addresses, + "raylet_socket_names": raylet_socket_names, + # Web UI should be running. + "webui_url": _webui_url_helper(redis_client) + } def get_address_info_from_redis(redis_address, node_ip_address, num_retries=5, - use_raylet=True, redis_password=None): counter = 0 while True: try: return get_address_info_from_redis_helper( - redis_address, - node_ip_address, - use_raylet=use_raylet, - redis_password=redis_password) + redis_address, node_ip_address, redis_password=redis_password) except Exception: if counter == num_retries: raise @@ -1414,7 +1279,6 @@ def _init(address_info=None, plasma_directory=None, huge_pages=False, include_webui=True, - use_raylet=None, plasma_store_socket_name=None, raylet_socket_name=None, temp_dir=None): @@ -1474,7 +1338,6 @@ def _init(address_info=None, Store with hugetlbfs support. Requires plasma_directory. include_webui: Boolean flag indicating whether to start the web UI, which is a Jupyter notebook. - use_raylet: True if the new raylet code path should be used. plasma_store_socket_name (str): If provided, it will specify the socket name used by the plasma store. raylet_socket_name (str): If provided, it will specify the socket path @@ -1497,16 +1360,6 @@ def _init(address_info=None, else: driver_mode = SCRIPT_MODE - if use_raylet is None: - if os.environ.get("RAY_USE_XRAY") == "0": - # This environment variable is used in our testing setup. - logger.info("Detected environment variable 'RAY_USE_XRAY' with " - "value {}. This turns OFF xray.".format( - os.environ.get("RAY_USE_XRAY"))) - use_raylet = False - else: - use_raylet = True - # Get addresses of existing services. if address_info is None: address_info = {} @@ -1561,7 +1414,6 @@ def _init(address_info=None, plasma_directory=plasma_directory, huge_pages=huge_pages, include_webui=include_webui, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) @@ -1610,10 +1462,7 @@ def _init(address_info=None, node_ip_address = services.get_node_ip_address(redis_address) # Get the address info of the processes to connect to from Redis. address_info = get_address_info_from_redis( - redis_address, - node_ip_address, - use_raylet=use_raylet, - redis_password=redis_password) + redis_address, node_ip_address, redis_password=redis_password) # Connect this driver to Redis, the object store, and the local scheduler. # Choose the first object store and local scheduler if there are multiple. @@ -1625,18 +1474,11 @@ def _init(address_info=None, driver_address_info = { "node_ip_address": node_ip_address, "redis_address": address_info["redis_address"], - "store_socket_name": ( - address_info["object_store_addresses"][0].name), + "store_socket_name": address_info["object_store_addresses"][0], "webui_url": address_info["webui_url"] } - if not use_raylet: - driver_address_info["manager_socket_name"] = ( - address_info["object_store_addresses"][0].manager_name) - driver_address_info["local_scheduler_socket_name"] = ( - address_info["local_scheduler_socket_names"][0]) - else: - driver_address_info["raylet_socket_name"] = ( - address_info["raylet_socket_names"][0]) + driver_address_info["raylet_socket_name"] = ( + address_info["raylet_socket_names"][0]) # We only pass `temp_dir` to a worker (WORKER_MODE). # It can't be a worker here. @@ -1645,7 +1487,6 @@ def _init(address_info=None, object_id_seed=object_id_seed, mode=driver_mode, worker=global_worker, - use_raylet=use_raylet, redis_password=redis_password) return address_info @@ -1669,7 +1510,6 @@ def init(redis_address=None, plasma_directory=None, huge_pages=False, include_webui=True, - use_raylet=None, configure_logging=True, logging_level=logging.INFO, logging_format=ray_constants.LOGGER_FORMAT, @@ -1736,7 +1576,6 @@ def init(redis_address=None, Store with hugetlbfs support. Requires plasma_directory. include_webui: Boolean flag indicating whether to start the web UI, which is a Jupyter notebook. - use_raylet: True if the new raylet code path should be used. configure_logging: True if allow the logging cofiguration here. Otherwise, the users may want to configure it by their own. logging_level: Logging level, default will be loging.INFO. @@ -1767,22 +1606,6 @@ def init(redis_address=None, else: raise Exception("Perhaps you called ray.init twice by accident?") - if use_raylet is None: - if os.environ.get("RAY_USE_XRAY") == "0": - # This environment variable is used in our testing setup. - logger.info("Detected environment variable 'RAY_USE_XRAY' with " - "value {}. This turns OFF xray.".format( - os.environ.get("RAY_USE_XRAY"))) - use_raylet = False - else: - use_raylet = True - - if not use_raylet and redis_password is not None: - raise Exception("Setting the 'redis_password' argument is not " - "supported in legacy Ray. To run Ray with " - "password-protected Redis ports, set " - "'use_raylet=True'.") - # Convert hostnames to numerical IP address. if node_ip_address is not None: node_ip_address = services.address_to_ip(node_ip_address) @@ -1809,7 +1632,6 @@ def init(redis_address=None, huge_pages=huge_pages, include_webui=include_webui, object_store_memory=object_store_memory, - use_raylet=use_raylet, plasma_store_socket_name=plasma_store_socket_name, raylet_socket_name=raylet_socket_name, temp_dir=temp_dir) @@ -1887,9 +1709,6 @@ def print_error_messages_raylet(worker): This runs in a separate thread on the driver and prints error messages in the background. """ - if not worker.use_raylet: - raise Exception("This function is specific to the raylet code path.") - worker.error_message_pubsub_client = worker.redis_client.pubsub( ignore_subscribe_messages=True) # Exports that are published after the call to @@ -2004,7 +1823,6 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker, - use_raylet=True, redis_password=None): """Connect this worker to the local scheduler, to Plasma, and to Redis. @@ -2015,7 +1833,6 @@ def connect(info, deterministic. mode: The mode of the worker. One of SCRIPT_MODE, WORKER_MODE, and LOCAL_MODE. - use_raylet: True if the new raylet code path should be used. redis_password (str): Prevents external clients without the password from connecting to Redis if provided. """ @@ -2038,7 +1855,6 @@ def connect(info, worker.actor_id = NIL_ACTOR_ID worker.connected = True worker.set_mode(mode) - worker.use_raylet = use_raylet # If running Ray in LOCAL_MODE, there is no need to create call # create_worker or to start the worker service. @@ -2067,7 +1883,6 @@ def connect(info, traceback_str = traceback.format_exc() ray.utils.push_error_to_driver_through_redis( worker.redis_client, - worker.use_raylet, ray_constants.VERSION_MISMATCH_PUSH_ERROR, traceback_str, driver_id=None) @@ -2108,7 +1923,6 @@ def connect(info, "driver_id": worker.worker_id, "start_time": time.time(), "plasma_store_socket": info["store_socket_name"], - "plasma_manager_socket": info.get("manager_socket_name"), "local_scheduler_socket": info.get("local_scheduler_socket_name"), "raylet_socket": info.get("raylet_socket_name") } @@ -2123,7 +1937,6 @@ def connect(info, worker_dict = { "node_ip_address": worker.node_ip_address, "plasma_store_socket": info["store_socket_name"], - "plasma_manager_socket": info["manager_socket_name"], "local_scheduler_socket": info["local_scheduler_socket_name"] } if redirect_worker_output: @@ -2135,18 +1948,10 @@ def connect(info, raise Exception("This code should be unreachable.") # Create an object store client. - if not worker.use_raylet: - worker.plasma_client = thread_safe_client( - plasma.connect(info["store_socket_name"], - info["manager_socket_name"], 64)) - else: - worker.plasma_client = thread_safe_client( - plasma.connect(info["store_socket_name"], "", 64)) + worker.plasma_client = thread_safe_client( + plasma.connect(info["store_socket_name"], "", 64)) - if not worker.use_raylet: - local_scheduler_socket = info["local_scheduler_socket_name"] - else: - local_scheduler_socket = info["raylet_socket_name"] + local_scheduler_socket = info["raylet_socket_name"] # If this is a driver, set the current task ID, the task driver ID, and set # the task index to 0. @@ -2177,28 +1982,22 @@ def connect(info, # rerun the driver. nil_actor_counter = 0 - driver_task = ray.local_scheduler.Task( - worker.task_driver_id, ray.ObjectID(NIL_FUNCTION_ID), [], 0, - worker.current_task_id, worker.task_index, - ray.ObjectID(NIL_ACTOR_ID), ray.ObjectID(NIL_ACTOR_ID), - ray.ObjectID(NIL_ACTOR_ID), ray.ObjectID(NIL_ACTOR_ID), - nil_actor_counter, False, [], {"CPU": 0}, {}, worker.use_raylet) + driver_task = ray.raylet.Task(worker.task_driver_id, + ray.ObjectID(NIL_FUNCTION_ID), [], 0, + worker.current_task_id, + worker.task_index, + ray.ObjectID(NIL_ACTOR_ID), + ray.ObjectID(NIL_ACTOR_ID), + ray.ObjectID(NIL_ACTOR_ID), + ray.ObjectID(NIL_ACTOR_ID), + nil_actor_counter, [], {"CPU": 0}, {}) # Add the driver task to the task table. - if not worker.use_raylet: - global_state._execute_command( - driver_task.task_id(), "RAY.TASK_TABLE_ADD", - driver_task.task_id().id(), TASK_STATUS_RUNNING, - NIL_LOCAL_SCHEDULER_ID, - driver_task.execution_dependencies_string(), 0, - ray.local_scheduler.task_to_string(driver_task)) - else: - global_state._execute_command( - driver_task.task_id(), "RAY.TABLE_ADD", - ray.gcs_utils.TablePrefix.RAYLET_TASK, - ray.gcs_utils.TablePubsub.RAYLET_TASK, - driver_task.task_id().id(), - driver_task._serialized_raylet_task()) + global_state._execute_command(driver_task.task_id(), "RAY.TABLE_ADD", + ray.gcs_utils.TablePrefix.RAYLET_TASK, + ray.gcs_utils.TablePubsub.RAYLET_TASK, + driver_task.task_id().id(), + driver_task._serialized_raylet_task()) # Set the driver's current task ID to the task ID assigned to the # driver task. @@ -2207,9 +2006,9 @@ def connect(info, # A non-driver worker begins without an assigned task. worker.current_task_id = ray.ObjectID(NIL_ID) - worker.local_scheduler_client = ray.local_scheduler.LocalSchedulerClient( + worker.local_scheduler_client = ray.raylet.LocalSchedulerClient( local_scheduler_socket, worker.worker_id, is_worker, - worker.current_task_id, worker.use_raylet) + worker.current_task_id) # Start the import thread import_thread.ImportThread(worker, mode).start() @@ -2221,16 +2020,10 @@ def connect(info, # temporarily using this implementation which constantly queries the # scheduler for new error messages. if mode == SCRIPT_MODE: - if not worker.use_raylet: - t = threading.Thread( - target=print_error_messages, - name="ray_print_error_messages", - args=(worker, )) - else: - t = threading.Thread( - target=print_error_messages_raylet, - name="ray_print_error_messages", - args=(worker, )) + t = threading.Thread( + target=print_error_messages_raylet, + name="ray_print_error_messages", + args=(worker, )) # Making the thread a daemon causes it to exit when the main thread # exits. t.daemon = True @@ -2238,7 +2031,7 @@ def connect(info, # If we are using the raylet code path and we are not in local mode, start # a background thread to periodically flush profiling data to the GCS. - if mode != LOCAL_MODE and worker.use_raylet: + if mode != LOCAL_MODE: worker.profiler.start_flush_thread() if mode == SCRIPT_MODE: @@ -2395,6 +2188,9 @@ def register_custom_serializer(cls, # worker and not across workers. class_id = random_string() + # Make sure class_id is a string. + class_id = ray.utils.binary_to_hex(class_id) + if driver_id is None: driver_id_bytes = worker.task_driver_id.id() else: @@ -2481,7 +2277,7 @@ def put(value, worker=global_worker): # In LOCAL_MODE, ray.put is the identity operation. return value object_id = worker.local_scheduler_client.compute_put_id( - worker.current_task_id, worker.put_index, worker.use_raylet) + worker.current_task_id, worker.put_index) worker.put_object(object_id, value) worker.put_index += 1 return object_id @@ -2554,21 +2350,8 @@ def wait(object_ids, num_returns=1, timeout=None, worker=global_worker): raise Exception("num_returns cannot be greater than the number " "of objects provided to ray.wait.") timeout = timeout if timeout is not None else 2**30 - if worker.use_raylet: - ready_ids, remaining_ids = worker.local_scheduler_client.wait( - object_ids, num_returns, timeout, False) - else: - object_id_strs = [ - plasma.ObjectID(object_id.id()) for object_id in object_ids - ] - ready_ids, remaining_ids = worker.plasma_client.wait( - object_id_strs, timeout, num_returns) - ready_ids = [ - ray.ObjectID(object_id.binary()) for object_id in ready_ids - ] - remaining_ids = [ - ray.ObjectID(object_id.binary()) for object_id in remaining_ids - ] + ready_ids, remaining_ids = worker.local_scheduler_client.wait( + object_ids, num_returns, timeout, False) return ready_ids, remaining_ids diff --git a/python/ray/workers/default_worker.py b/python/ray/workers/default_worker.py index 7fe46218f..4ec9e4d14 100644 --- a/python/ray/workers/default_worker.py +++ b/python/ray/workers/default_worker.py @@ -88,10 +88,7 @@ if __name__ == "__main__": tempfile_services.set_temp_root(args.temp_dir) ray.worker.connect( - info, - mode=ray.WORKER_MODE, - use_raylet=(args.raylet_name is not None), - redis_password=args.redis_password) + info, mode=ray.WORKER_MODE, redis_password=args.redis_password) error_explanation = """ This error is unexpected and should not have happened. Somehow a worker diff --git a/python/setup.py b/python/setup.py index 29e296a13..1636ead05 100644 --- a/python/setup.py +++ b/python/setup.py @@ -19,13 +19,10 @@ import setuptools.command.build_ext as _build_ext # NOTE: The lists below must be kept in sync with ray/CMakeLists.txt. ray_files = [ - "ray/core/src/common/thirdparty/redis/src/redis-server", - "ray/core/src/common/redis_module/libray_redis_module.so", + "ray/core/src/ray/thirdparty/redis/src/redis-server", + "ray/core/src/ray/gcs/redis_module/libray_redis_module.so", "ray/core/src/plasma/plasma_store_server", - "ray/core/src/plasma/plasma_manager", - "ray/core/src/local_scheduler/local_scheduler", - "ray/core/src/local_scheduler/liblocal_scheduler_library_python.so", - "ray/core/src/global_scheduler/global_scheduler", + "ray/core/src/ray/raylet/liblocal_scheduler_library_python.so", "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet", "ray/WebUI.ipynb" ] diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt deleted file mode 100644 index b024b4a04..000000000 --- a/src/common/CMakeLists.txt +++ /dev/null @@ -1,131 +0,0 @@ -cmake_minimum_required(VERSION 3.4) - -project(common) - -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - include_directories("${CMAKE_CURRENT_LIST_DIR}/lib/python") -endif () - -add_subdirectory(redis_module) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -g") - -include_directories(thirdparty/ae) - -# Compile flatbuffers - -set(COMMON_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs") -set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/) - -set(COMMON_FBS_OUTPUT_FILES - "${OUTPUT_DIR}/common_generated.h") - -add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES}) - -add_custom_command( - OUTPUT ${COMMON_FBS_OUTPUT_FILES} - # The --gen-object-api flag generates a C++ class MessageT for each - # flatbuffers message Message, which can be used to store deserialized - # messages in data structures. This is currently used for ObjectInfo for - # example. - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${COMMON_FBS_SRC} --gen-object-api --scoped-enums - DEPENDS ${FBS_DEPENDS} - COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}" - VERBATIM) - -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES}) - - # Generate Python bindings for the flatbuffers objects. - set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/) - add_custom_command( - TARGET gen_common_python_fbs - COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC} - DEPENDS ${FBS_DEPENDS} - COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}" - VERBATIM) - - # Encode the fact that the ray redis module requires the autogenerated - # flatbuffer files to compile. - add_dependencies(ray_redis_module gen_common_python_fbs) - - add_dependencies(gen_common_python_fbs flatbuffers_ep) -endif() - -if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") - add_custom_target(gen_common_java_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES}) - - # Generate Java bindings for the flatbuffers objects. - set(JAVA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/java) - add_custom_command( - TARGET gen_common_java_fbs - COMMAND ${FLATBUFFERS_COMPILER} -j -o ${JAVA_OUTPUT_DIR} ${COMMON_FBS_SRC} - DEPENDS ${FBS_DEPENDS} - COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}" - VERBATIM) - - # Encode the fact that the ray redis module requires the autogenerated - # flatbuffer files to compile. - add_dependencies(ray_redis_module gen_common_java_fbs) - - add_dependencies(gen_common_java_fbs flatbuffers_ep) -endif() - -add_custom_target( - hiredis - COMMAND make - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis) - -add_library(common STATIC - event_loop.cc - common.cc - common_protocol.cc - task.cc - io.cc - net.cc - logging.cc - state/redis.cc - state/table.cc - state/object_table.cc - state/task_table.cc - state/db_client_table.cc - state/driver_table.cc - state/actor_notification_table.cc - state/local_scheduler_table.cc - state/error_table.cc - thirdparty/ae/ae.c - thirdparty/sha256.c) - -add_dependencies(common arrow) - -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - add_dependencies(common gen_common_python_fbs) -endif() - -if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") - add_dependencies(common gen_common_java_fbs) -endif() - -target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a") - -function(define_test test_name library) - add_executable(${test_name} test/${test_name}.cc ${ARGN}) - add_dependencies(${test_name} hiredis flatbuffers_ep) - target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${library} -lpthread) - target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DLOCAL_SCHEDULER_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4") -endfunction() - -define_test(db_tests "") -define_test(io_tests "") -define_test(task_tests "") -define_test(redis_tests "") -define_test(task_table_tests "") -define_test(object_table_tests "") - -add_custom_target(copy_redis ALL) -foreach(file "redis-cli" "redis-server") -add_custom_command(TARGET copy_redis POST_BUILD - COMMAND ${CMAKE_COMMAND} -E - copy ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/redis/src/${file} - ${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file}) -endforeach() diff --git a/src/common/common.cc b/src/common/common.cc deleted file mode 100644 index 0a6da6a29..000000000 --- a/src/common/common.cc +++ /dev/null @@ -1,20 +0,0 @@ -#include "common.h" - -#include -#include -#include -#include -#include -#include - -#include "io.h" -#include - -const unsigned char NIL_DIGEST[DIGEST_SIZE] = {0}; - -int64_t current_time_ms() { - std::chrono::milliseconds ms_since_epoch = - std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()); - return ms_since_epoch.count(); -} diff --git a/src/common/common.h b/src/common/common.h deleted file mode 100644 index f95bfcca5..000000000 --- a/src/common/common.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef COMMON_H -#define COMMON_H - -#include -#include -#include -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif -#include -#include -#ifndef _WIN32 -#include -#endif - -#ifdef __cplusplus -#include -extern "C" { -#endif -#include "sha256.h" -#ifdef __cplusplus -} -#endif - -#include "arrow/util/macros.h" -#include "plasma/common.h" -#include "ray/id.h" -#include "ray/util/logging.h" - -#include "state/ray_config.h" - -/** Definitions for Ray logging levels. */ -#define RAY_COMMON_DEBUG 0 -#define RAY_COMMON_INFO 1 -#define RAY_COMMON_WARNING 2 -#define RAY_COMMON_ERROR 3 -#define RAY_COMMON_FATAL 4 - -/** - * RAY_COMMON_LOG_LEVEL should be defined to one of the above logging level - * integer values. Any logging statement in the code with a logging level - * greater than or equal to RAY_COMMON_LOG_LEVEL will be outputted to stderr. - * The default logging level is INFO. */ -#ifndef RAY_COMMON_LOG_LEVEL -#define RAY_COMMON_LOG_LEVEL RAY_COMMON_INFO -#endif - -/* These are exit codes for common errors that can occur in Ray components. */ -#define EXIT_COULD_NOT_BIND_PORT -2 - -/** This macro indicates that this pointer owns the data it is pointing to - * and is responsible for freeing it. */ -#define OWNER - -/** The worker ID is the ID of a worker or driver. */ -typedef ray::UniqueID WorkerID; - -typedef ray::UniqueID DBClientID; - -#define MAX(x, y) ((x) >= (y) ? (x) : (y)) -#define MIN(x, y) ((x) <= (y) ? (x) : (y)) - -/** Definitions for computing hash digests. */ -#define DIGEST_SIZE SHA256_BLOCK_SIZE - -extern const unsigned char NIL_DIGEST[DIGEST_SIZE]; - -/** - * Return the current time in milliseconds since the Unix epoch. - * - * @return The number of milliseconds since the Unix epoch. - */ -int64_t current_time_ms(); - -#endif diff --git a/src/common/doc/tasks.md b/src/common/doc/tasks.md deleted file mode 100644 index 4431afae2..000000000 --- a/src/common/doc/tasks.md +++ /dev/null @@ -1,32 +0,0 @@ -# Task specifications, task instances and task logs - -A *task specification* contains all information that is needed for computing -the results of a task: - -- The ID of the task -- The function ID of the function that executes the task -- The arguments (either object IDs for pass by reference -or values for pass by value) -- The IDs of the result objects - -From these, a task ID can be computed which is also stored in the task -specification. - -A *task* represents the execution of a task specification. -It consists of: - -- A scheduling state (WAITING, SCHEDULED, RUNNING, DONE) -- The target node where the task is scheduled or executed -- The task specification - -The task data structures are defined in `common/task.h`. - -The *task table* is a mapping from the task ID to the *task* information. It is -updated by various parts of the system: - -1. The local scheduler writes it with status WAITING when submits a task to the global scheduler -2. The global scheduler appends an update WAITING -> SCHEDULED together with the node ID when assigning the task to a local scheduler -3. The local scheduler appends an update SCHEDULED -> RUNNING when it assigns a task to a worker -4. The local scheduler appends an update RUNNING -> DONE when the task finishes execution - -The task table is defined in `common/state/task_table.h`. diff --git a/src/common/event_loop.cc b/src/common/event_loop.cc deleted file mode 100644 index e3d9cc4a2..000000000 --- a/src/common/event_loop.cc +++ /dev/null @@ -1,63 +0,0 @@ -#include "event_loop.h" - -#include "common.h" -#include - -#define INITIAL_EVENT_LOOP_SIZE 1024 - -event_loop *event_loop_create(void) { - return aeCreateEventLoop(INITIAL_EVENT_LOOP_SIZE); -} - -void event_loop_destroy(event_loop *loop) { - /* Clean up timer events. This is to make valgrind happy. */ - aeTimeEvent *te = loop->timeEventHead; - while (te) { - aeTimeEvent *next = te->next; - free(te); - te = next; - } - aeDeleteEventLoop(loop); -} - -bool event_loop_add_file(event_loop *loop, - int fd, - int events, - event_loop_file_handler handler, - void *context) { - /* Try to add the file descriptor. */ - int err = aeCreateFileEvent(loop, fd, events, handler, context); - /* If it cannot be added, increase the size of the event loop. */ - if (err == AE_ERR && errno == ERANGE) { - err = aeResizeSetSize(loop, 3 * aeGetSetSize(loop) / 2); - if (err != AE_OK) { - return false; - } - err = aeCreateFileEvent(loop, fd, events, handler, context); - } - /* In any case, test if there were errors. */ - return (err == AE_OK); -} - -void event_loop_remove_file(event_loop *loop, int fd) { - aeDeleteFileEvent(loop, fd, EVENT_LOOP_READ | EVENT_LOOP_WRITE); -} - -int64_t event_loop_add_timer(event_loop *loop, - int64_t timeout, - event_loop_timer_handler handler, - void *context) { - return aeCreateTimeEvent(loop, timeout, handler, context, NULL); -} - -int event_loop_remove_timer(event_loop *loop, int64_t id) { - return aeDeleteTimeEvent(loop, id); -} - -void event_loop_run(event_loop *loop) { - aeMain(loop); -} - -void event_loop_stop(event_loop *loop) { - aeStop(loop); -} diff --git a/src/common/event_loop.h b/src/common/event_loop.h deleted file mode 100644 index e489ab4fb..000000000 --- a/src/common/event_loop.h +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef EVENT_LOOP_H -#define EVENT_LOOP_H - -#include - -extern "C" { -#ifdef _WIN32 -/* Quirks mean that Windows version needs to be included differently */ -#include -#include -#else -#include "ae/ae.h" -#endif -} - -/* Unique timer ID that will be generated when the timer is added to the - * event loop. Will not be reused later on in another call - * to event_loop_add_timer. */ -typedef long long timer_id; - -typedef aeEventLoop event_loop; - -/* File descriptor is readable. */ -#define EVENT_LOOP_READ AE_READABLE - -/* File descriptor is writable. */ -#define EVENT_LOOP_WRITE AE_WRITABLE - -/* Constant specifying that the timer is done and it will be removed. */ -#define EVENT_LOOP_TIMER_DONE AE_NOMORE - -/* Signature of the handler that will be called when there is a new event - * on the file descriptor that this handler has been registered for. The - * context is the one that was passed into add_file by the user. The - * events parameter indicates which event is available on the file, - * it can be EVENT_LOOP_READ or EVENT_LOOP_WRITE. */ -typedef void (*event_loop_file_handler)(event_loop *loop, - int fd, - void *context, - int events); - -/* This handler will be called when a timer times out. The id of the timer - * as well as the context that was specified when registering this handler - * are passed as arguments. The return is the number of milliseconds the - * timer shall be reset to or EVENT_LOOP_TIMER_DONE if the timer shall - * not be triggered again. */ -typedef int (*event_loop_timer_handler)(event_loop *loop, - timer_id timer_id, - void *context); - -/* Create and return a new event loop. */ -event_loop *event_loop_create(void); - -/* Deallocate space associated with the event loop that was created - * with the "create" function. */ -void event_loop_destroy(event_loop *loop); - -/* Register a handler that will be called any time a new event happens on - * a file descriptor. Can specify a context that will be passed as an - * argument to the handler. Currently there can only be one handler per file. - * The events parameter specifies which events we listen to: EVENT_LOOP_READ - * or EVENT_LOOP_WRITE. */ -bool event_loop_add_file(event_loop *loop, - int fd, - int events, - event_loop_file_handler handler, - void *context); - -/* Remove a registered file event handler from the event loop. */ -void event_loop_remove_file(event_loop *loop, int fd); - -/** Register a handler that will be called after a time slice of - * "timeout" milliseconds. - * - * @param loop The event loop. - * @param timeout The timeout in milliseconds. - * @param handler The handler for the timeout. - * @param context User context that can be passed in and will be passed in - * as an argument for the timer handler. - * @return The ID of the timer. - */ -int64_t event_loop_add_timer(event_loop *loop, - int64_t timeout, - event_loop_timer_handler handler, - void *context); - -/** - * Remove a registered time event handler from the event loop. Can be called - * multiple times on the same timer. - * - * @param loop The event loop. - * @param timer_id The ID of the timer to be removed. - * @return Returns 0 if the removal was successful. - */ -int event_loop_remove_timer(event_loop *loop, int64_t timer_id); - -/* Run the event loop. */ -void event_loop_run(event_loop *loop); - -/* Stop the event loop. */ -void event_loop_stop(event_loop *loop); - -#endif diff --git a/src/common/format/common.fbs b/src/common/format/common.fbs deleted file mode 100644 index a5b2177f1..000000000 --- a/src/common/format/common.fbs +++ /dev/null @@ -1,203 +0,0 @@ - -// Indices into resource vectors. -// A resource vector maps a resource index to the number -// of units of that resource required. - -table Arg { - // Object ID for pass-by-reference arguments. Normally there is only one - // object ID in this list which represents the object that is being passed. - // However to support reducers in a MapReduce workload, we also support - // passing multiple object IDs for each argument. - object_ids: [string]; - // Data for pass-by-value arguments. - data: string; -} - -table ResourcePair { - // The name of the resource. - key: string; - // The quantity of the resource. - value: double; -} - -// NOTE: This enum is duplicate with the `Language` enum in `gcs.fbs`, -// because we cannot include this file in `gcs.fbs` due to cyclic dependency. -// TODO(raulchen): remove it once we get rid of legacy ray. -enum TaskLanguage:int { - PYTHON = 0, - JAVA = 1 -} - -table TaskInfo { - // ID of the driver that created this task. - driver_id: string; - // Task ID of the task. - task_id: string; - // Task ID of the parent task. - parent_task_id: string; - // A count of the number of tasks submitted by the parent task before this one. - parent_counter: int; - // The ID of the actor to create if this is an actor creation task. - actor_creation_id: string; - // The dummy object ID of the actor creation task if this is an actor method. - actor_creation_dummy_object_id: string; - // Actor ID of the task. This is the actor that this task is executed on - // or NIL_ACTOR_ID if the task is just a normal task. - actor_id: string; - // The ID of the handle that was used to submit the task. This should be - // unique across handles with the same actor_id. - actor_handle_id: string; - // Number of tasks that have been submitted to this actor so far. - actor_counter: int; - // True if this task is an actor checkpoint task and false otherwise. - is_actor_checkpoint_method: bool; - // Function ID of the task. - function_id: string; - // Task arguments. - args: [Arg]; - // Object IDs of return values. - returns: [string]; - // The required_resources vector indicates the quantities of the different - // resources required by this task. - required_resources: [ResourcePair]; - // The resources required for placing this task on a node. If this is empty, - // then the placement resources are equal to the required_resources. - required_placement_resources: [ResourcePair]; - // The language that this task belongs to - language: TaskLanguage; - // Function descriptor, which is a list of strings that can - // uniquely describe a function. - // For a Python function, it should be: [module_name, class_name, function_name] - // For a Java function, it should be: [class_name, method_name, type_descriptor] - // TODO(hchen): after changing Python worker to use function_descriptor, - // function_id can be removed. - function_descriptor: [string]; -} - -// Object information data structure. -// NOTE(pcm): This structure is replicated in -// https://github.com/apache/arrow/blob/master/cpp/src/plasma/format/common.fbs, -// so if you modify it, you should also modify that one. -table ObjectInfo { - // Object ID of this object. - object_id: string; - // Number of bytes the content of this object occupies in memory. - data_size: long; - // Number of bytes the metadata of this object occupies in memory. - metadata_size: long; - // Number of clients using the objects. - ref_count: int; - // Unix epoch of when this object was created. - create_time: long; - // How long creation of this object took. - construct_duration: long; - // Hash of the object content. If the object is not sealed yet this is - // an empty string. - digest: string; - // Specifies if this object was deleted or added. - is_deletion: bool; -} - -root_type TaskInfo; - -table TaskExecutionDependencies { - // A list of object IDs representing this task's dependencies at execution - // time. - execution_dependencies: [string]; -} - -root_type TaskExecutionDependencies; - -table SubscribeToNotificationsReply { - // The object ID of the object that the notification is about. - object_id: string; - // The size of the object. - object_size: long; - // The IDs of the managers that contain this object. - manager_ids: [string]; -} - -root_type SubscribeToNotificationsReply; - -table TaskReply { - // The task ID of the task that the message is about. - task_id: string; - // The state of the task. This is encoded as a bit mask of scheduling_state - // enum values in task.h. - state: long; - // A local scheduler ID. - local_scheduler_id: string; - // A string of bytes representing the task's TaskExecutionDependencies. - execution_dependencies: string; - // A string of bytes representing the task specification. - task_spec: string; - // The number of times the task was spilled back by local schedulers. - spillback_count: long; - // A boolean representing whether the update was successful. This field - // should only be used for test-and-set operations. - updated: bool; -} - -root_type TaskReply; - -table SubscribeToDBClientTableReply { - // The db client ID of the client that the message is about. - db_client_id: string; - // The type of the client. - client_type: string; - // If the client is a local scheduler, this is the address of the plasma - // manager that the local scheduler is connected to. Otherwise, it is empty. - manager_address: string; - // True if the message is about the addition of a client and false if it is - // about the deletion of a client. - is_insertion: bool; -} - -root_type SubscribeToDBClientTableReply; - -table LocalSchedulerInfoMessage { - // The db client ID of the client that the message is about. - db_client_id: string; - // The total number of workers that are connected to this local scheduler. - total_num_workers: long; - // The number of tasks queued in this local scheduler. - task_queue_length: long; - // The number of workers that are available and waiting for tasks. - available_workers: long; - // The resources generally available to this local scheduler. - static_resources: [ResourcePair]; - // The resources currently available to this local scheduler. - dynamic_resources: [ResourcePair]; - // Whether the local scheduler is dead. If true, then all other fields - // besides `db_client_id` will not be set. - is_dead: bool; -} - -root_type LocalSchedulerInfoMessage; - -table ResultTableReply { - // The task ID of the task that created the object. - task_id: string; - // Whether the task created the object through a ray.put. - is_put: bool; - // The size of the object created. - data_size: long; - // The hash of the object created. - hash: string; -} - -root_type ResultTableReply; - -table DriverTableMessage { - // The driver ID of the driver that died. - driver_id: string; -} - -table ActorCreationNotification { - // The ID of the actor that was created. - actor_id: string; - // The ID of the driver that created the actor. - driver_id: string; - // The ID of the local scheduler that created the actor. - local_scheduler_id: string; -} diff --git a/src/common/io.cc b/src/common/io.cc deleted file mode 100644 index 1999b7054..000000000 --- a/src/common/io.cc +++ /dev/null @@ -1,416 +0,0 @@ -#include "io.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common.h" -#include "event_loop.h" - -#ifndef _WIN32 -/* This function is actually not declared in standard POSIX, so declare it. */ -extern int usleep(useconds_t usec); -#endif - -int bind_inet_sock(const int port, bool shall_listen) { - struct sockaddr_in name; - int socket_fd = socket(PF_INET, SOCK_STREAM, 0); - if (socket_fd < 0) { - RAY_LOG(ERROR) << "socket() failed for port " << port; - return -1; - } - name.sin_family = AF_INET; - name.sin_port = htons(port); - name.sin_addr.s_addr = htonl(INADDR_ANY); - int on = 1; - /* TODO(pcm): http://stackoverflow.com/q/1150635 */ - if (ioctl(socket_fd, FIONBIO, (char *) &on) < 0) { - RAY_LOG(ERROR) << "ioctl failed"; - close(socket_fd); - return -1; - } - int *const pon = (int *const) & on; - if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, pon, sizeof(on)) < 0) { - RAY_LOG(ERROR) << "setsockopt failed for port " << port; - close(socket_fd); - return -1; - } - if (bind(socket_fd, (struct sockaddr *) &name, sizeof(name)) < 0) { - RAY_LOG(ERROR) << "Bind failed for port " << port; - close(socket_fd); - return -1; - } - if (shall_listen && listen(socket_fd, 128) == -1) { - RAY_LOG(ERROR) << "Could not listen to socket " << port; - close(socket_fd); - return -1; - } - return socket_fd; -} - -int bind_ipc_sock(const char *socket_pathname, bool shall_listen) { - struct sockaddr_un socket_address; - int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (socket_fd < 0) { - RAY_LOG(ERROR) << "socket() failed for pathname " << socket_pathname; - return -1; - } - /* Tell the system to allow the port to be reused. */ - int on = 1; - if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, - sizeof(on)) < 0) { - RAY_LOG(ERROR) << "setsockopt failed for pathname " << socket_pathname; - close(socket_fd); - return -1; - } - - unlink(socket_pathname); - memset(&socket_address, 0, sizeof(socket_address)); - socket_address.sun_family = AF_UNIX; - if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { - RAY_LOG(ERROR) << "Socket pathname is too long."; - close(socket_fd); - return -1; - } - strncpy(socket_address.sun_path, socket_pathname, - strlen(socket_pathname) + 1); - - if (bind(socket_fd, (struct sockaddr *) &socket_address, - sizeof(socket_address)) != 0) { - RAY_LOG(ERROR) << "Bind failed for pathname " << socket_pathname; - close(socket_fd); - return -1; - } - if (shall_listen && listen(socket_fd, 128) == -1) { - RAY_LOG(ERROR) << "Could not listen to socket " << socket_pathname; - close(socket_fd); - return -1; - } - return socket_fd; -} - -int connect_ipc_sock_retry(const char *socket_pathname, - int num_retries, - int64_t timeout) { - /* Pick the default values if the user did not specify. */ - if (num_retries < 0) { - num_retries = RayConfig::instance().num_connect_attempts(); - } - if (timeout < 0) { - timeout = RayConfig::instance().connect_timeout_milliseconds(); - } - - RAY_CHECK(socket_pathname); - int fd = -1; - for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) { - fd = connect_ipc_sock(socket_pathname); - if (fd >= 0) { - break; - } - if (num_attempts == 0) { - RAY_LOG(ERROR) << "Connection to socket failed for pathname " - << socket_pathname; - } - /* Sleep for timeout milliseconds. */ - usleep(timeout * 1000); - } - /* If we could not connect to the socket, exit. */ - if (fd == -1) { - RAY_LOG(FATAL) << "Could not connect to socket " << socket_pathname; - } - return fd; -} - -int connect_ipc_sock(const char *socket_pathname) { - struct sockaddr_un socket_address; - int socket_fd; - - socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (socket_fd < 0) { - RAY_LOG(ERROR) << "socket() failed for pathname " << socket_pathname; - return -1; - } - - memset(&socket_address, 0, sizeof(socket_address)); - socket_address.sun_family = AF_UNIX; - if (strlen(socket_pathname) + 1 > sizeof(socket_address.sun_path)) { - RAY_LOG(ERROR) << "Socket pathname is too long."; - return -1; - } - strncpy(socket_address.sun_path, socket_pathname, - strlen(socket_pathname) + 1); - - if (connect(socket_fd, (struct sockaddr *) &socket_address, - sizeof(socket_address)) != 0) { - close(socket_fd); - return -1; - } - - return socket_fd; -} - -int connect_inet_sock_retry(const char *ip_addr, - int port, - int num_retries, - int64_t timeout) { - /* Pick the default values if the user did not specify. */ - if (num_retries < 0) { - num_retries = RayConfig::instance().num_connect_attempts(); - } - if (timeout < 0) { - timeout = RayConfig::instance().connect_timeout_milliseconds(); - } - - RAY_CHECK(ip_addr); - int fd = -1; - for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) { - fd = connect_inet_sock(ip_addr, port); - if (fd >= 0) { - break; - } - if (num_attempts == 0) { - RAY_LOG(ERROR) << "Connection to socket failed for address " << ip_addr - << ":" << port; - } - /* Sleep for timeout milliseconds. */ - usleep(timeout * 1000); - } - /* If we could not connect to the socket, exit. */ - if (fd == -1) { - RAY_LOG(FATAL) << "Could not connect to address " << ip_addr << ":" << port; - } - return fd; -} - -int connect_inet_sock(const char *ip_addr, int port) { - int fd = socket(PF_INET, SOCK_STREAM, 0); - if (fd < 0) { - RAY_LOG(ERROR) << "socket() failed for address " << ip_addr << ":" << port; - return -1; - } - - struct hostent *manager = gethostbyname(ip_addr); /* TODO(pcm): cache this */ - if (!manager) { - RAY_LOG(ERROR) << "Failed to get hostname from address " << ip_addr << ":" - << port; - close(fd); - return -1; - } - - struct sockaddr_in addr; - addr.sin_family = AF_INET; - memcpy(&addr.sin_addr.s_addr, manager->h_addr_list[0], manager->h_length); - addr.sin_port = htons(port); - - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) != 0) { - close(fd); - return -1; - } - return fd; -} - -int accept_client(int socket_fd) { - int client_fd = accept(socket_fd, NULL, NULL); - if (client_fd < 0) { - RAY_LOG(ERROR) << "Error reading from socket."; - return -1; - } - return client_fd; -} - -int write_bytes(int fd, uint8_t *cursor, size_t length) { - ssize_t nbytes = 0; - size_t bytesleft = length; - size_t offset = 0; - while (bytesleft > 0) { - /* While we haven't written the whole message, write to the file - * descriptor, advance the cursor, and decrease the amount left to write. */ - nbytes = write(fd, cursor + offset, bytesleft); - if (nbytes < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { - continue; - } - return -1; /* Errno will be set. */ - } else if (0 == nbytes) { - /* Encountered early EOF. */ - return -1; - } - RAY_CHECK(nbytes > 0); - bytesleft -= nbytes; - offset += nbytes; - } - - return 0; -} - -int do_write_message(int fd, int64_t type, int64_t length, uint8_t *bytes) { - int64_t version = RayConfig::instance().ray_protocol_version(); - int closed; - closed = write_bytes(fd, (uint8_t *) &version, sizeof(version)); - if (closed) { - return closed; - } - closed = write_bytes(fd, (uint8_t *) &type, sizeof(type)); - if (closed) { - return closed; - } - closed = write_bytes(fd, (uint8_t *) &length, sizeof(length)); - if (closed) { - return closed; - } - closed = write_bytes(fd, bytes, length * sizeof(char)); - if (closed) { - return closed; - } - return 0; -} - -int write_message(int fd, - int64_t type, - int64_t length, - uint8_t *bytes, - std::mutex *mutex) { - if (mutex != NULL) { - std::unique_lock guard(*mutex); - return do_write_message(fd, type, length, bytes); - } else { - return do_write_message(fd, type, length, bytes); - } -} - -int read_bytes(int fd, uint8_t *cursor, size_t length) { - ssize_t nbytes = 0; - /* Termination condition: EOF or read 'length' bytes total. */ - size_t bytesleft = length; - size_t offset = 0; - while (bytesleft > 0) { - nbytes = read(fd, cursor + offset, bytesleft); - if (nbytes < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { - continue; - } - return -1; /* Errno will be set. */ - } else if (0 == nbytes) { - /* Encountered early EOF. */ - return -1; - } - RAY_CHECK(nbytes > 0); - bytesleft -= nbytes; - offset += nbytes; - } - - return 0; -} - -void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes) { - int64_t version; - int closed = read_bytes(fd, (uint8_t *) &version, sizeof(version)); - if (closed) { - goto disconnected; - } - RAY_CHECK(version == RayConfig::instance().ray_protocol_version()); - closed = read_bytes(fd, (uint8_t *) type, sizeof(*type)); - if (closed) { - goto disconnected; - } - closed = read_bytes(fd, (uint8_t *) length, sizeof(*length)); - if (closed) { - goto disconnected; - } - *bytes = (uint8_t *) malloc(*length * sizeof(uint8_t)); - closed = read_bytes(fd, *bytes, *length); - if (closed) { - free(*bytes); - goto disconnected; - } - return; - -disconnected: - /* Handle the case in which the socket is closed. */ - *type = static_cast(CommonMessageType::DISCONNECT_CLIENT); - *length = 0; - *bytes = NULL; - return; -} - -uint8_t *read_message_async(event_loop *loop, int sock) { - int64_t size; - int error = read_bytes(sock, (uint8_t *) &size, sizeof(int64_t)); - if (error < 0) { - /* The other side has closed the socket. */ - RAY_LOG(DEBUG) << "Socket has been closed, or some other error has " - << "occurred."; - if (loop != NULL) { - event_loop_remove_file(loop, sock); - } - close(sock); - return NULL; - } - uint8_t *message = (uint8_t *) malloc(size); - error = read_bytes(sock, message, size); - if (error < 0) { - /* The other side has closed the socket. */ - RAY_LOG(DEBUG) << "Socket has been closed, or some other error has " - << "occurred."; - if (loop != NULL) { - event_loop_remove_file(loop, sock); - } - close(sock); - return NULL; - } - return message; -} - -int64_t read_vector(int fd, int64_t *type, std::vector &buffer) { - int64_t version; - int closed = read_bytes(fd, (uint8_t *) &version, sizeof(version)); - if (closed) { - goto disconnected; - } - RAY_CHECK(version == RayConfig::instance().ray_protocol_version()); - int64_t length; - closed = read_bytes(fd, (uint8_t *) type, sizeof(*type)); - if (closed) { - goto disconnected; - } - closed = read_bytes(fd, (uint8_t *) &length, sizeof(length)); - if (closed) { - goto disconnected; - } - if (static_cast(length) > buffer.size()) { - buffer.resize(length); - } - closed = read_bytes(fd, buffer.data(), length); - if (closed) { - goto disconnected; - } - return length; -disconnected: - /* Handle the case in which the socket is closed. */ - *type = static_cast(CommonMessageType::DISCONNECT_CLIENT); - return 0; -} - -void write_log_message(int fd, const char *message) { - /* Account for the \0 at the end of the string. */ - do_write_message(fd, static_cast(CommonMessageType::LOG_MESSAGE), - strlen(message) + 1, (uint8_t *) message); -} - -char *read_log_message(int fd) { - uint8_t *bytes; - int64_t type; - int64_t length; - read_message(fd, &type, &length, &bytes); - RAY_CHECK(static_cast(type) == - CommonMessageType::LOG_MESSAGE); - return (char *) bytes; -} diff --git a/src/common/io.h b/src/common/io.h deleted file mode 100644 index 3f976445a..000000000 --- a/src/common/io.h +++ /dev/null @@ -1,228 +0,0 @@ -#ifndef IO_H -#define IO_H - -#include -#include - -#include -#include - -struct aeEventLoop; -typedef aeEventLoop event_loop; - -enum class CommonMessageType : int32_t { - /** Disconnect a client. */ - DISCONNECT_CLIENT, - /** Log a message from a client. */ - LOG_MESSAGE, - /** Submit a task to the local scheduler. */ - SUBMIT_TASK, -}; - -/* Helper functions for socket communication. */ - -/** - * Binds to an Internet socket at the given port. Removes any existing file at - * the pathname. Returns a non-blocking file descriptor for the socket, or -1 - * if an error occurred. - * - * @note Since the returned file descriptor is non-blocking, it is not - * recommended to use the Linux read and write calls directly, since these - * might read or write a partial message. Instead, use the provided - * write_message and read_message methods. - * - * @param port The port to bind to. - * @param shall_listen Are we also starting to listen on the socket? - * @return A non-blocking file descriptor for the socket, or -1 if an error - * occurs. - */ -int bind_inet_sock(const int port, bool shall_listen); - -/** - * Binds to a Unix domain streaming socket at the given - * pathname. Removes any existing file at the pathname. - * - * @param socket_pathname The pathname for the socket. - * @param shall_listen Are we also starting to listen on the socket? - * @return A blocking file descriptor for the socket, or -1 if an error - * occurs. - */ -int bind_ipc_sock(const char *socket_pathname, bool shall_listen); - -/** - * Connect to a Unix domain streaming socket at the given - * pathname. - * - * @param socket_pathname The pathname for the socket. - * @return A file descriptor for the socket, or -1 if an error occurred. - */ -int connect_ipc_sock(const char *socket_pathname); - -/** - * Connect to a Unix domain streaming socket at the given - * pathname, or fail after some number of retries. - * - * @param socket_pathname The pathname for the socket. - * @param num_retries The number of times to retry the connection - * before exiting. If -1 is provided, then this defaults to - * num_connect_attempts. - * @param timeout The number of milliseconds to wait in between - * retries. If -1 is provided, then this defaults to - * connect_timeout_milliseconds. - * @return A file descriptor for the socket, or -1 if an error occurred. - */ -int connect_ipc_sock_retry(const char *socket_pathname, - int num_retries, - int64_t timeout); - -/** - * Connect to an Internet socket at the given address and port. - * - * @param ip_addr The IP address to connect to. - * @param port The port number to connect to. - * - * @param socket_pathname The pathname for the socket. - * @return A file descriptor for the socket, or -1 if an error occurred. - */ -int connect_inet_sock(const char *ip_addr, int port); - -/** - * Connect to an Internet socket at the given address and port, or fail after - * some number of retries. - * - * @param ip_addr The IP address to connect to. - * @param port The port number to connect to. - * @param num_retries The number of times to retry the connection - * before exiting. If -1 is provided, then this defaults to - * num_connect_attempts. - * @param timeout The number of milliseconds to wait in between - * retries. If -1 is provided, then this defaults to - * connect_timeout_milliseconds. - * @return A file descriptor for the socket, or -1 if an error occurred. - */ -int connect_inet_sock_retry(const char *ip_addr, - int port, - int num_retries, - int64_t timeout); - -/** - * Accept a new client connection on the given socket - * descriptor. Returns a descriptor for the new socket. - */ -int accept_client(int socket_fd); - -/* Reading and writing data. */ - -/** - * Write a sequence of bytes on a file descriptor. The bytes should then be read - * by read_message. - * - * @param fd The file descriptor to write to. It can be non-blocking. - * @param version The protocol version. - * @param type The type of the message to send. - * @param length The size in bytes of the bytes parameter. - * @param bytes The address of the message to send. - * @param mutex If not NULL, the whole write operation will be locked - * with this mutex, otherwise do nothing. - * @return int Whether there was an error while writing. 0 corresponds to - * success and -1 corresponds to an error (errno will be set). - */ -int write_message(int fd, - int64_t type, - int64_t length, - uint8_t *bytes, - std::mutex *mutex = NULL); - -/** - * Read a sequence of bytes written by write_message from a file descriptor. - * This allocates space for the message. - * - * @note The caller must free the memory. - * - * @param fd The file descriptor to read from. It can be non-blocking. - * @param type The type of the message that is read will be written at this - * address. If there was an error while reading, this will be - * DISCONNECT_CLIENT. - * @param length The size in bytes of the message that is read will be written - * at this address. This size does not include the bytes used to encode - * the type and length. If there was an error while reading, this will - * be 0. - * @param bytes The address at which to write the pointer to the bytes that are - * read and allocated by this function. If there was an error while - * reading, this will be NULL. - * @return Void. - */ -void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes); - -/** - * Read a message from a file descriptor and remove the file descriptor from the - * event loop if there is an error. This will actually do two reads. The first - * read reads sizeof(int64_t) bytes to determine the number of bytes to read in - * the next read. - * - * @param loop: The event loop. - * @param sock: The file descriptor to read from. - * @return A byte buffer contining the message or NULL if there was an - * error. The buffer needs to be freed by the user. - */ -uint8_t *read_message_async(event_loop *loop, int sock); - -/** - * Read a sequence of bytes written by write_message from a file descriptor. - * This does not allocate space for the message if the provided buffer is - * large enough and can therefore often avoid allocations. - * - * @param fd The file descriptor to read from. It can be non-blocking. - * @param type The type of the message that is read will be written at this - * address. If there was an error while reading, this will be - * DISCONNECT_CLIENT. - * @param buffer The array the message will be written to. If it is not - * large enough to hold the message, it will be enlarged by read_vector. - * @return Number of bytes of the message that were read. This size does not - * include the bytes used to encode the type and length. If there was - * an error while reading, this will be 0. - */ -int64_t read_vector(int fd, int64_t *type, std::vector &buffer); - -/** - * Write a null-terminated string to a file descriptor. - */ -void write_log_message(int fd, const char *message); - -/** - * Reads a null-terminated string from the file descriptor that has been - * written by write_log_message. Allocates and returns a pointer to the string. - * NOTE: Caller must free the memory! - */ -char *read_log_message(int fd); - -/** - * Read a sequence of bytes from a file descriptor into a buffer. This will - * block until one of the following happens: (1) there is an error (2) end of - * file, or (3) all length bytes have been written. - * - * @note The buffer pointed to by cursor must already have length number of - * bytes allocated before calling this method. - * - * @param fd The file descriptor to read from. It can be non-blocking. - * @param cursor The cursor pointing to the beginning of the buffer. - * @param length The size of the byte sequence to read. - * @return int Whether there was an error while reading. 0 corresponds to - * success and -1 corresponds to an error (errno will be set). - */ -int read_bytes(int fd, uint8_t *cursor, size_t length); - -/** - * Write a sequence of bytes into a file descriptor. This will block until one - * of the following happens: (1) there is an error (2) end of file, or (3) all - * length bytes have been written. - * - * @param fd The file descriptor to write to. It can be non-blocking. - * @param cursor The cursor pointing to the beginning of the bytes to send. - * @param length The size of the bytes sequence to write. - * @return int Whether there was an error while writing. 0 corresponds to - * success and -1 corresponds to an error (errno will be set). - */ -int write_bytes(int fd, uint8_t *cursor, size_t length); - -#endif /* IO_H */ diff --git a/src/common/logging.cc b/src/common/logging.cc deleted file mode 100644 index 9802dd3d0..000000000 --- a/src/common/logging.cc +++ /dev/null @@ -1,107 +0,0 @@ -#include "logging.h" - -#include -#include -#include - -#include - -#include "state/redis.h" -#include "io.h" -#include -#include - -static const char *log_levels[5] = {"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}; -static const char *log_fmt = - "HMSET log:%s:%s log_level %s event_type %s message %s timestamp %s"; - -struct RayLoggerImpl { - /* String that identifies this client type. */ - const char *client_type; - /* Suppress all log messages below this level. */ - int log_level; - /* Whether or not we have a direct connection to Redis. */ - int is_direct; - /* Either a db_handle or a socket to a process with a db_handle, - * depending on the is_direct flag. */ - void *conn; -}; - -RayLogger *RayLogger_init(const char *client_type, - int log_level, - int is_direct, - void *conn) { - RayLogger *logger = (RayLogger *) malloc(sizeof(RayLogger)); - logger->client_type = client_type; - logger->log_level = log_level; - logger->is_direct = is_direct; - logger->conn = conn; - return logger; -} - -void RayLogger_free(RayLogger *logger) { - free(logger); -} - -void RayLogger_log(RayLogger *logger, - int log_level, - const char *event_type, - const char *message) { - if (log_level < logger->log_level) { - return; - } - if (log_level < RAY_LOG_DEBUG || log_level > RAY_LOG_FATAL) { - return; - } - struct timeval tv; - gettimeofday(&tv, NULL); - std::string timestamp = - std::to_string(tv.tv_sec) + "." + std::to_string(tv.tv_usec); - - /* Find number of bytes that would have been written for formatted_message - * size */ - size_t formatted_message_size = - std::snprintf(nullptr, 0, log_fmt, timestamp.c_str(), "%b", - log_levels[log_level], event_type, message, - timestamp.c_str()) + - 1; - /* Fill out everything except the client ID, which is binary data. */ - char formatted_message[formatted_message_size]; - std::snprintf(formatted_message, formatted_message_size, log_fmt, - timestamp.c_str(), "%b", log_levels[log_level], event_type, - message, timestamp.c_str()); - - if (logger->is_direct) { - DBHandle *db = (DBHandle *) logger->conn; - /* Fill in the client ID and send the message to Redis. */ - - redisAsyncContext *context = get_redis_context(db, db->client); - - int status = - redisAsyncCommand(context, NULL, NULL, formatted_message, - (char *) db->client.data(), sizeof(db->client)); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error while logging message to log table"); - } - } else { - /* If we don't own a Redis connection, we leave our client - * ID to be filled in by someone else. */ - int *socket_fd = (int *) logger->conn; - write_log_message(*socket_fd, formatted_message); - } -} - -void RayLogger_log_event(DBHandle *db, - uint8_t *key, - int64_t key_length, - uint8_t *value, - int64_t value_length, - double timestamp) { - std::string timestamp_string = std::to_string(timestamp); - int status = redisAsyncCommand(db->context, NULL, NULL, "ZADD %b %s %b", key, - key_length, timestamp_string.c_str(), value, - value_length); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, "error while logging message to event log"); - } -} diff --git a/src/common/logging.h b/src/common/logging.h deleted file mode 100644 index 1fa57a60c..000000000 --- a/src/common/logging.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef LOGGING_H -#define LOGGING_H - -#define RAY_LOG_VERBOSE -1 -#define RAY_LOG_DEBUG 0 -#define RAY_LOG_INFO 1 -#define RAY_LOG_WARNING 2 -#define RAY_LOG_ERROR 3 -#define RAY_LOG_FATAL 4 - -/* Entity types. */ -#define RAY_FUNCTION "FUNCTION" -#define RAY_OBJECT "OBJECT" -#define RAY_TASK "TASK" - -#include "state/db.h" - -typedef struct RayLoggerImpl RayLogger; - -/* Initialize a Ray logger for the given client type and logging level. If the - * is_direct flag is set, the logger will treat the given connection as a - * direct connection to the log. Otherwise, it will treat it as a socket to - * another process with a connection to the log. - * NOTE: User is responsible for freeing the returned logger. */ -RayLogger *RayLogger_init(const char *client_type, - int log_level, - int is_direct, - void *conn); - -/* Free the logger. This does not free the connection to the log. */ -void RayLogger_free(RayLogger *logger); - -/* Log an event at the given log level with the given event_type. - * NOTE: message cannot contain spaces! JSON format is recommended. - * TODO: Support spaces in messages. */ -void RayLogger_log(RayLogger *logger, - int log_level, - const char *event_type, - const char *message); - -/** - * Log an event to the event log. - * - * @param db The database handle. - * @param key The key in Redis to store the event in. - * @param key_length The length of the key. - * @param value The value to log. - * @param value_length The length of the value. - * @return Void. - */ -void RayLogger_log_event(DBHandle *db, - uint8_t *key, - int64_t key_length, - uint8_t *value, - int64_t value_length, - double time); - -#endif /* LOGGING_H */ diff --git a/src/common/net.cc b/src/common/net.cc deleted file mode 100644 index 3f2aaf6fa..000000000 --- a/src/common/net.cc +++ /dev/null @@ -1,24 +0,0 @@ -#include "net.h" - -#include - -#include - -#include "common.h" - -int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port) { - char port_str[6]; - int parsed = sscanf(ip_addr_port, "%15[0-9.]:%5[0-9]", ip_addr, port_str); - if (parsed != 2) { - return -1; - } - *port = atoi(port_str); - return 0; -} - -/* Return true if the ip address is valid. */ -bool valid_ip_address(const std::string &ip_address) { - struct sockaddr_in sa; - int result = inet_pton(AF_INET, ip_address.c_str(), &sa.sin_addr); - return result == 1; -} diff --git a/src/common/net.h b/src/common/net.h deleted file mode 100644 index 109cdf3fa..000000000 --- a/src/common/net.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef NET_H -#define NET_H - -/* Helper function to parse a string of the form : into the - * given ip_addr and port pointers. The ip_addr buffer must already be - * allocated. Return 0 upon success and -1 upon failure. */ -int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port); - -#endif /* NET_H */ diff --git a/src/common/redis_module/ray_redis_module.cc b/src/common/redis_module/ray_redis_module.cc deleted file mode 100644 index d594f74ef..000000000 --- a/src/common/redis_module/ray_redis_module.cc +++ /dev/null @@ -1,1886 +0,0 @@ -#include - -#include "common_protocol.h" -#include "format/common_generated.h" -#include "ray/gcs/format/gcs_generated.h" -#include "ray/id.h" -#include "redis_string.h" -#include "redismodule.h" -#include "task.h" - -#if RAY_USE_NEW_GCS -// Under this flag, ray-project/credis will be loaded. Specifically, via -// "path/redis-server --loadmodule --loadmodule " (dlopen() under the hood) will a definition of "module" -// be supplied. -// -// All commands in this file that depend on "module" must be wrapped by "#if -// RAY_USE_NEW_GCS", until we switch to this launch configuration as the -// default. -#include "chain_module.h" -extern RedisChainModule module; -#endif - -// Various tables are maintained in redis: -// -// == OBJECT TABLE == -// -// This consists of two parts: -// - The object location table, indexed by OL:object_id, which is the set of -// plasma manager indices that have access to the object. -// (In redis this is represented by a zset (sorted set).) -// -// - The object info table, indexed by OI:object_id, which is a hashmap of: -// "hash" -> the hash of the object, -// "data_size" -> the size of the object in bytes, -// "task" -> the task ID that generated this object. -// "is_put" -> 0 or 1. -// -// == TASK TABLE == -// -// It maps each TT:task_id to a hash: -// "state" -> the state of the task, encoded as a bit mask of scheduling_state -// enum values in task.h, -// "local_scheduler_id" -> the ID of the local scheduler the task is assigned -// to, -// "TaskSpec" -> serialized bytes of a TaskInfo (defined in common.fbs), which -// describes the details this task. -// -// See also the definition of TaskReply in common.fbs. - -#define OBJECT_INFO_PREFIX "OI:" -#define OBJECT_LOCATION_PREFIX "OL:" -#define OBJECT_NOTIFICATION_PREFIX "ON:" -#define TASK_PREFIX "TT:" -#define OBJECT_BCAST "BCAST" - -#define OBJECT_CHANNEL_PREFIX "OC:" - -#define CHECK_ERROR(STATUS, MESSAGE) \ - if ((STATUS) == REDISMODULE_ERR) { \ - return RedisModule_ReplyWithError(ctx, (MESSAGE)); \ - } - -/// Parse a Redis string into a TablePubsub channel. -TablePubsub ParseTablePubsub(const RedisModuleString *pubsub_channel_str) { - long long pubsub_channel_long; - RAY_CHECK(RedisModule_StringToLongLong( - pubsub_channel_str, &pubsub_channel_long) == REDISMODULE_OK) - << "Pubsub channel must be a valid TablePubsub"; - auto pubsub_channel = static_cast(pubsub_channel_long); - RAY_CHECK(pubsub_channel >= TablePubsub::MIN && - pubsub_channel <= TablePubsub::MAX) - << "Pubsub channel must be a valid TablePubsub"; - return pubsub_channel; -} - -/// Format a pubsub channel for a specific key. pubsub_channel_str should -/// contain a valid TablePubsub. -RedisModuleString *FormatPubsubChannel( - RedisModuleCtx *ctx, - const RedisModuleString *pubsub_channel_str, - const RedisModuleString *id) { - // Format the pubsub channel enum to a string. TablePubsub_MAX should be more - // than enough digits, but add 1 just in case for the null terminator. - char pubsub_channel[static_cast(TablePubsub::MAX) + 1]; - sprintf(pubsub_channel, "%d", - static_cast(ParseTablePubsub(pubsub_channel_str))); - return RedisString_Format(ctx, "%s:%S", pubsub_channel, id); -} - -// TODO(swang): This helper function should be deprecated by the version below, -// which uses enums for table prefixes. -RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx, - const char *prefix, - RedisModuleString *keyname, - int mode, - RedisModuleString **mutated_key_str) { - RedisModuleString *prefixed_keyname = - RedisString_Format(ctx, "%s%S", prefix, keyname); - // Pass out the key being mutated, should the caller request so. - if (mutated_key_str != nullptr) { - *mutated_key_str = prefixed_keyname; - } - RedisModuleKey *key = - (RedisModuleKey *) RedisModule_OpenKey(ctx, prefixed_keyname, mode); - return key; -} - -RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx, - RedisModuleString *prefix_enum, - RedisModuleString *keyname, - int mode, - RedisModuleString **mutated_key_str) { - long long prefix_long; - RAY_CHECK(RedisModule_StringToLongLong(prefix_enum, &prefix_long) == - REDISMODULE_OK) - << "Prefix must be a valid TablePrefix"; - auto prefix = static_cast(prefix_long); - RAY_CHECK(prefix != TablePrefix::UNUSED) - << "This table has no prefix registered"; - RAY_CHECK(prefix >= TablePrefix::MIN && prefix <= TablePrefix::MAX) - << "Prefix must be a valid TablePrefix"; - return OpenPrefixedKey(ctx, EnumNameTablePrefix(prefix), keyname, mode, - mutated_key_str); -} - -RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx, - const char *prefix, - RedisModuleString *keyname, - int mode) { - return OpenPrefixedKey(ctx, prefix, keyname, mode, - /*mutated_key_str=*/nullptr); -} - -RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx, - RedisModuleString *prefix_enum, - RedisModuleString *keyname, - int mode) { - return OpenPrefixedKey(ctx, prefix_enum, keyname, mode, - /*mutated_key_str=*/nullptr); -} - -/// Open the key used to store the channels that should be published to when an -/// update happens at the given keyname. -RedisModuleKey *OpenBroadcastKey(RedisModuleCtx *ctx, - RedisModuleString *pubsub_channel_str, - RedisModuleString *keyname, - int mode) { - RedisModuleString *channel = - FormatPubsubChannel(ctx, pubsub_channel_str, keyname); - RedisModuleString *prefixed_keyname = - RedisString_Format(ctx, "BCAST:%S", channel); - RedisModuleKey *key = - (RedisModuleKey *) RedisModule_OpenKey(ctx, prefixed_keyname, mode); - return key; -} - -/** - * This is a helper method to convert a redis module string to a flatbuffer - * string. - * - * @param fbb The flatbuffer builder. - * @param redis_string The redis string. - * @return The flatbuffer string. - */ -flatbuffers::Offset RedisStringToFlatbuf( - flatbuffers::FlatBufferBuilder &fbb, - RedisModuleString *redis_string) { - size_t redis_string_size; - const char *redis_string_str = - RedisModule_StringPtrLen(redis_string, &redis_string_size); - return fbb.CreateString(redis_string_str, redis_string_size); -} - -/** - * Publish a notification to a client's notification channel about an insertion - * or deletion to the db client table. - * - * TODO(swang): Use flatbuffers for the notification message. - * The format for the published notification is: - * : - * If no manager address is provided, manager_address will be set to ":". If - * is_insertion is true, then the last field will be "1", else "0". - * - * @param ctx The Redis context. - * @param ray_client_id The ID of the database client that was inserted or - * deleted. - * @param client_type The type of client that was inserted or deleted. - * @param manager_address An optional secondary address for the object manager - * associated with this database client. - * @param is_insertion A boolean that's true if the update was an insertion and - * false if deletion. - * @return True if the publish was successful and false otherwise. - */ -bool PublishDBClientNotification(RedisModuleCtx *ctx, - RedisModuleString *ray_client_id, - RedisModuleString *client_type, - RedisModuleString *manager_address, - bool is_insertion) { - /* Construct strings to publish on the db client channel. */ - RedisModuleString *channel_name = - RedisModule_CreateString(ctx, "db_clients", strlen("db_clients")); - /* Construct the flatbuffers object to publish over the channel. */ - flatbuffers::FlatBufferBuilder fbb; - /* Use an empty aux address if one is not passed in. */ - flatbuffers::Offset manager_address_str; - if (manager_address != NULL) { - manager_address_str = RedisStringToFlatbuf(fbb, manager_address); - } else { - manager_address_str = fbb.CreateString("", strlen("")); - } - /* Create the flatbuffers message. */ - auto message = CreateSubscribeToDBClientTableReply( - fbb, RedisStringToFlatbuf(fbb, ray_client_id), - RedisStringToFlatbuf(fbb, client_type), manager_address_str, - is_insertion); - fbb.Finish(message); - /* Create a Redis string to publish by serializing the flatbuffers object. */ - RedisModuleString *client_info = RedisModule_CreateString( - ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize()); - - /* Publish the client info on the db client channel. */ - RedisModuleCallReply *reply; - reply = RedisModule_Call(ctx, "PUBLISH", "ss", channel_name, client_info); - return (reply != NULL); -} - -/** - * Register a client with Redis. This is called from a client with the command: - * - * RAY.CONNECT - * ... - * - * The command can take an arbitrary number of pairs of field names and keys, - * and these will be stored in a hashmap associated with this client. Several - * fields are singled out for special treatment: - * - * manager_address: This is provided by local schedulers and plasma - * managers and should be the address of the plasma manager that the - * client is associated with. This is published to the "db_clients" - * channel by the RAY.CONNECT command. - * - * @param ray_client_id The db client ID of the client. - * @param node_ip_address The IP address of the node the client is on. - * @param client_type The type of the client (e.g., plasma_manager). - * @return OK if the operation was successful. - */ -int Connect_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc < 4) { - return RedisModule_WrongArity(ctx); - } - if (argc % 2 != 0) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *ray_client_id = argv[1]; - RedisModuleString *node_ip_address = argv[2]; - RedisModuleString *client_type = argv[3]; - - /* Add this client to the Ray db client table. */ - RedisModuleKey *db_client_table_key = - OpenPrefixedKey(ctx, DB_CLIENT_PREFIX, ray_client_id, REDISMODULE_WRITE); - - if (RedisModule_KeyType(db_client_table_key) != REDISMODULE_KEYTYPE_EMPTY) { - return RedisModule_ReplyWithError(ctx, "Client already exists"); - } - - /* This will be used to construct a publish message. */ - RedisModuleString *manager_address = NULL; - RedisModuleString *manager_address_key = RedisModule_CreateString( - ctx, "manager_address", strlen("manager_address")); - RedisModuleString *deleted = RedisModule_CreateString(ctx, "0", strlen("0")); - - RedisModule_HashSet(db_client_table_key, REDISMODULE_HASH_CFIELDS, - "ray_client_id", ray_client_id, "node_ip_address", - node_ip_address, "client_type", client_type, "deleted", - deleted, NULL); - - for (int i = 4; i < argc; i += 2) { - RedisModuleString *key = argv[i]; - RedisModuleString *value = argv[i + 1]; - RedisModule_HashSet(db_client_table_key, REDISMODULE_HASH_NONE, key, value, - NULL); - if (RedisModule_StringCompare(key, manager_address_key) == 0) { - manager_address = value; - } - } - /* Clean up. */ - if (!PublishDBClientNotification(ctx, ray_client_id, client_type, - manager_address, true)) { - return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful"); - } - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -/** - * Remove a client from Redis. This is called from a client with the command: - * - * RAY.DISCONNECT - * - * This method also publishes a notification to all subscribers to the - * db_clients channel. The notification consists of a message of the form ":". - * - * @param ray_client_id The db client ID of the client. - * @return OK if the operation was successful. - */ -int Disconnect_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 2) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *ray_client_id = argv[1]; - - /* Get the client type. */ - RedisModuleKey *db_client_table_key = - OpenPrefixedKey(ctx, DB_CLIENT_PREFIX, ray_client_id, REDISMODULE_WRITE); - - RedisModuleString *deleted_string; - RedisModule_HashGet(db_client_table_key, REDISMODULE_HASH_CFIELDS, "deleted", - &deleted_string, NULL); - long long deleted; - int parsed = RedisModule_StringToLongLong(deleted_string, &deleted); - if (parsed != REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "Unable to parse deleted field"); - } - - bool published = true; - if (deleted == 0) { - /* Remove the client from the client table. */ - RedisModuleString *deleted = - RedisModule_CreateString(ctx, "1", strlen("1")); - RedisModule_HashSet(db_client_table_key, REDISMODULE_HASH_CFIELDS, - "deleted", deleted, NULL); - - RedisModuleString *client_type; - RedisModuleString *manager_address; - RedisModule_HashGet(db_client_table_key, REDISMODULE_HASH_CFIELDS, - "client_type", &client_type, "manager_address", - &manager_address, NULL); - - /* Publish the deletion notification on the db client channel. */ - published = PublishDBClientNotification(ctx, ray_client_id, client_type, - manager_address, false); - } - - if (!published) { - /* Return an error message if we weren't able to publish the deletion - * notification. */ - return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful"); - } - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -/** - * Lookup an entry in the object table. - * - * This is called from a client with the command: - * - * RAY.OBJECT_TABLE_LOOKUP - * - * @param object_id A string representing the object ID. - * @return A list, possibly empty, of plasma manager IDs that are listed in the - * object table as having the object. If there was no entry found in - * the object table, returns nil. - */ -int ObjectTableLookup_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 2) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleKey *key = - OpenPrefixedKey(ctx, OBJECT_LOCATION_PREFIX, argv[1], REDISMODULE_READ); - - if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) { - /* Return nil if no entry was found. */ - return RedisModule_ReplyWithNull(ctx); - } - if (RedisModule_ValueLength(key) == 0) { - /* Return empty list if there are no managers. */ - return RedisModule_ReplyWithArray(ctx, 0); - } - - CHECK_ERROR( - RedisModule_ZsetFirstInScoreRange(key, REDISMODULE_NEGATIVE_INFINITE, - REDISMODULE_POSITIVE_INFINITE, 1, 1), - "Unable to initialize zset iterator"); - - RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN); - int num_results = 0; - do { - RedisModuleString *curr = RedisModule_ZsetRangeCurrentElement(key, NULL); - RedisModule_ReplyWithString(ctx, curr); - num_results += 1; - } while (RedisModule_ZsetRangeNext(key)); - RedisModule_ReplySetArrayLength(ctx, num_results); - - return REDISMODULE_OK; -} - -/** - * Publish a notification to a client's object notification channel if at least - * one manager is listed as having the object in the object table. - * - * @param ctx The Redis context. - * @param client_id The ID of the client that is being notified. - * @param object_id The object ID of interest. - * @param key The opened key for the entry in the object table corresponding to - * the object ID of interest. - * @return True if the publish was successful and false otherwise. - */ -bool PublishObjectNotification(RedisModuleCtx *ctx, - RedisModuleString *client_id, - RedisModuleString *object_id, - RedisModuleString *data_size, - RedisModuleKey *key) { - flatbuffers::FlatBufferBuilder fbb; - - long long data_size_value; - if (RedisModule_StringToLongLong(data_size, &data_size_value) != - REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "data_size must be integer"); - } - - std::vector> manager_ids; - CHECK_ERROR( - RedisModule_ZsetFirstInScoreRange(key, REDISMODULE_NEGATIVE_INFINITE, - REDISMODULE_POSITIVE_INFINITE, 1, 1), - "Unable to initialize zset iterator"); - /* Loop over the managers in the object table for this object ID. */ - do { - RedisModuleString *curr = RedisModule_ZsetRangeCurrentElement(key, NULL); - manager_ids.push_back(RedisStringToFlatbuf(fbb, curr)); - } while (RedisModule_ZsetRangeNext(key)); - - auto message = CreateSubscribeToNotificationsReply( - fbb, RedisStringToFlatbuf(fbb, object_id), data_size_value, - fbb.CreateVector(manager_ids)); - fbb.Finish(message); - - /* Publish the notification to the clients notification channel. - * TODO(rkn): These notifications could be batched together. */ - RedisModuleString *channel_name = - RedisString_Format(ctx, "%s%S", OBJECT_CHANNEL_PREFIX, client_id); - - RedisModuleString *payload = RedisModule_CreateString( - ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize()); - - RedisModuleCallReply *reply; - reply = RedisModule_Call(ctx, "PUBLISH", "ss", channel_name, payload); - if (reply == NULL) { - return false; - } - return true; -} - -// NOTE(pcmoritz): This is a temporary redis command that will be removed once -// the GCS uses https://github.com/pcmoritz/credis. -int PublishTaskTableAdd(RedisModuleCtx *ctx, - RedisModuleString *id, - RedisModuleString *data) { - const char *buf = RedisModule_StringPtrLen(data, NULL); - auto message = flatbuffers::GetRoot(buf); - RAY_CHECK(message != nullptr); - - if (message->scheduling_state() == SchedulingState::WAITING || - message->scheduling_state() == SchedulingState::SCHEDULED) { - /* Build the PUBLISH topic and message for task table subscribers. The - * topic - * is a string in the format "TASK_PREFIX::". - * The - * message is a serialized SubscribeToTasksReply flatbuffer object. */ - std::string state = - std::to_string(static_cast(message->scheduling_state())); - RedisModuleString *publish_topic = RedisString_Format( - ctx, "%s%b:%s", TASK_PREFIX, message->scheduler_id()->str().data(), - sizeof(DBClientID), state.c_str()); - - /* Construct the flatbuffers object for the payload. */ - flatbuffers::FlatBufferBuilder fbb; - /* Create the flatbuffers message. */ - auto msg = - CreateTaskReply(fbb, RedisStringToFlatbuf(fbb, id), - static_cast(message->scheduling_state()), - fbb.CreateString(message->scheduler_id()), - fbb.CreateString(message->execution_dependencies()), - fbb.CreateString(message->task_info()), - message->spillback_count(), true /* not used */); - fbb.Finish(msg); - - RedisModuleString *publish_message = RedisModule_CreateString( - ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize()); - - RedisModuleCallReply *reply = - RedisModule_Call(ctx, "PUBLISH", "ss", publish_topic, publish_message); - - /* See how many clients received this publish. */ - long long num_clients = RedisModule_CallReplyInteger(reply); - RAY_CHECK(num_clients <= 1) << "Published to " << num_clients - << " clients."; - } - return RedisModule_ReplyWithSimpleString(ctx, "OK"); -} - -/// Publish a notification for a new entry at a key. This publishes a -/// notification to all subscribers of the table, as well as every client that -/// has requested notifications for this key. -/// -/// \param pubsub_channel_str The pubsub channel name that notifications for -/// this key should be published to. When publishing to a specific -/// client, the channel name should be :. -/// \param id The ID of the key that the notification is about. -/// \param data The data to publish. -/// \return OK if there is no error during a publish. -int PublishTableAdd(RedisModuleCtx *ctx, - RedisModuleString *pubsub_channel_str, - RedisModuleString *id, - RedisModuleString *data) { - // Serialize the notification to send. - flatbuffers::FlatBufferBuilder fbb; - auto data_flatbuf = RedisStringToFlatbuf(fbb, data); - auto message = CreateGcsTableEntry(fbb, RedisStringToFlatbuf(fbb, id), - fbb.CreateVector(&data_flatbuf, 1)); - fbb.Finish(message); - - // Write the data back to any subscribers that are listening to all table - // notifications. - RedisModuleCallReply *reply = - RedisModule_Call(ctx, "PUBLISH", "sb", pubsub_channel_str, - fbb.GetBufferPointer(), fbb.GetSize()); - if (reply == NULL) { - return RedisModule_ReplyWithError(ctx, "error during PUBLISH"); - } - - // Publish the data to any clients who requested notifications on this key. - RedisModuleKey *notification_key = OpenBroadcastKey( - ctx, pubsub_channel_str, id, REDISMODULE_READ | REDISMODULE_WRITE); - if (RedisModule_KeyType(notification_key) != REDISMODULE_KEYTYPE_EMPTY) { - // NOTE(swang): Sets are not implemented yet, so we use ZSETs instead. - CHECK_ERROR(RedisModule_ZsetFirstInScoreRange( - notification_key, REDISMODULE_NEGATIVE_INFINITE, - REDISMODULE_POSITIVE_INFINITE, 1, 1), - "Unable to initialize zset iterator"); - for (; !RedisModule_ZsetRangeEndReached(notification_key); - RedisModule_ZsetRangeNext(notification_key)) { - RedisModuleString *client_channel = - RedisModule_ZsetRangeCurrentElement(notification_key, NULL); - RedisModuleCallReply *reply = - RedisModule_Call(ctx, "PUBLISH", "sb", client_channel, - fbb.GetBufferPointer(), fbb.GetSize()); - if (reply == NULL) { - return RedisModule_ReplyWithError(ctx, "error during PUBLISH"); - } - } - } - return RedisModule_ReplyWithSimpleString(ctx, "OK"); -} - -// RAY.TABLE_ADD: -// TableAdd_RedisCommand: the actual command handler. -// (helper) TableAdd_DoWrite: performs the write to redis state. -// (helper) TableAdd_DoPublish: performs a publish after the write. -// ChainTableAdd_RedisCommand: the same command, chain-enabled. - -int TableAdd_DoWrite(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc, - RedisModuleString **mutated_key_str) { - if (argc != 5) { - return RedisModule_WrongArity(ctx); - } - RedisModuleString *prefix_str = argv[1]; - RedisModuleString *id = argv[3]; - RedisModuleString *data = argv[4]; - - RedisModuleKey *key = - OpenPrefixedKey(ctx, prefix_str, id, REDISMODULE_READ | REDISMODULE_WRITE, - mutated_key_str); - RedisModule_StringSet(key, data); - return REDISMODULE_OK; -} - -int TableAdd_DoPublish(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - if (argc != 5) { - return RedisModule_WrongArity(ctx); - } - RedisModuleString *pubsub_channel_str = argv[2]; - RedisModuleString *id = argv[3]; - RedisModuleString *data = argv[4]; - - TablePubsub pubsub_channel = ParseTablePubsub(pubsub_channel_str); - - if (pubsub_channel == TablePubsub::TASK) { - // Publish the task to its subscribers. - // TODO(swang): This is only necessary for legacy Ray and should be removed - // once we switch to using the new GCS API for the task table. - return PublishTaskTableAdd(ctx, id, data); - } else if (pubsub_channel != TablePubsub::NO_PUBLISH) { - // All other pubsub channels write the data back directly onto the channel. - return PublishTableAdd(ctx, pubsub_channel_str, id, data); - } else { - return RedisModule_ReplyWithSimpleString(ctx, "OK"); - } -} - -/// Add an entry at a key. This overwrites any existing data at the key. -/// Publishes a notification about the update to all subscribers, if a pubsub -/// channel is provided. -/// -/// This is called from a client with the command: -/// -/// RAY.TABLE_ADD -/// -/// \param table_prefix The prefix string for keys in this table. -/// \param pubsub_channel The pubsub channel name that notifications for -/// this key should be published to. When publishing to a specific -/// client, the channel name should be :. -/// \param id The ID of the key to set. -/// \param data The data to insert at the key. -/// \return The current value at the key, or OK if there is no value. -int TableAdd_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - TableAdd_DoWrite(ctx, argv, argc, /*mutated_key_str=*/nullptr); - return TableAdd_DoPublish(ctx, argv, argc); -} - -#if RAY_USE_NEW_GCS -int ChainTableAdd_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - return module.ChainReplicate(ctx, argv, argc, /*node_func=*/TableAdd_DoWrite, - /*tail_func=*/TableAdd_DoPublish); -} -#endif - -int TableAppend_DoWrite(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc, - RedisModuleString **mutated_key_str) { - if (argc < 5 || argc > 6) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *prefix_str = argv[1]; - RedisModuleString *id = argv[3]; - RedisModuleString *data = argv[4]; - RedisModuleString *index_str = nullptr; - if (argc == 6) { - index_str = argv[5]; - } - - // Set the keys in the table. - RedisModuleKey *key = - OpenPrefixedKey(ctx, prefix_str, id, REDISMODULE_READ | REDISMODULE_WRITE, - mutated_key_str); - // Determine the index at which the data should be appended. If no index is - // requested, then is the current length of the log. - size_t index = RedisModule_ValueLength(key); - if (index_str != nullptr) { - // Parse the requested index. - long long requested_index; - RAY_CHECK(RedisModule_StringToLongLong(index_str, &requested_index) == - REDISMODULE_OK); - RAY_CHECK(requested_index >= 0); - index = static_cast(requested_index); - } - // Only perform the append if the requested index matches the current length - // of the log, or if no index was requested. - if (index == RedisModule_ValueLength(key)) { - // The requested index matches the current length of the log or no index - // was requested. Perform the append. - int flags = REDISMODULE_ZADD_NX; - RedisModule_ZsetAdd(key, index, data, &flags); - // Check that we actually add a new entry during the append. This is only - // necessary since we implement the log with a sorted set, so all entries - // must be unique, or else we will have gaps in the log. - // TODO(rkn): We need to get rid of this uniqueness requirement. We can - // easily have multiple log events with the same message. - RAY_CHECK(flags == REDISMODULE_ZADD_ADDED) << "Appended a duplicate entry"; - return REDISMODULE_OK; - } else { - // The requested index did not match the current length of the log. Return - // an error message as a string. - static const char *reply = "ERR entry exists"; - RedisModule_ReplyWithStringBuffer(ctx, reply, strlen(reply)); - return REDISMODULE_ERR; - } -} - -int TableAppend_DoPublish(RedisModuleCtx *ctx, - RedisModuleString **argv, - int /*argc*/) { - RedisModuleString *pubsub_channel_str = argv[2]; - RedisModuleString *id = argv[3]; - RedisModuleString *data = argv[4]; - // Publish a message on the requested pubsub channel if necessary. - TablePubsub pubsub_channel = ParseTablePubsub(pubsub_channel_str); - if (pubsub_channel != TablePubsub::NO_PUBLISH) { - // All other pubsub channels write the data back directly onto the - // channel. - return PublishTableAdd(ctx, pubsub_channel_str, id, data); - } else { - return RedisModule_ReplyWithSimpleString(ctx, "OK"); - } -} - -/// Append an entry to the log stored at a key. Publishes a notification about -/// the update to all subscribers, if a pubsub channel is provided. -/// -/// This is called from a client with the command: -// -/// RAY.TABLE_APPEND -/// -/// -/// \param table_prefix The prefix string for keys in this table. -/// \param pubsub_channel The pubsub channel name that notifications for -/// this key should be published to. When publishing to a specific -/// client, the channel name should be :. -/// \param id The ID of the key to append to. -/// \param data The data to append to the key. -/// \param index If this is set, then the data must be appended at this index. -/// If the current log is shorter or longer than the requested index, -/// then the append will fail and an error message will be returned as a -/// string. -/// \return OK if the append succeeds, or an error message string if the append -/// fails. -int TableAppend_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - const int status = TableAppend_DoWrite(ctx, argv, argc, - /*mutated_key_str=*/nullptr); - if (status) { - return status; - } - return TableAppend_DoPublish(ctx, argv, argc); -} - -#if RAY_USE_NEW_GCS -int ChainTableAppend_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - return module.ChainReplicate(ctx, argv, argc, - /*node_func=*/TableAppend_DoWrite, - /*tail_func=*/TableAppend_DoPublish); -} -#endif - -/// A helper function to create and finish a GcsTableEntry, based on the -/// current value or values at the given key. -void TableEntryToFlatbuf(RedisModuleKey *table_key, - RedisModuleString *entry_id, - flatbuffers::FlatBufferBuilder &fbb) { - auto key_type = RedisModule_KeyType(table_key); - switch (key_type) { - case REDISMODULE_KEYTYPE_STRING: { - // Build the flatbuffer from the string data. - size_t data_len = 0; - char *data_buf = - RedisModule_StringDMA(table_key, &data_len, REDISMODULE_READ); - auto data = fbb.CreateString(data_buf, data_len); - auto message = CreateGcsTableEntry(fbb, RedisStringToFlatbuf(fbb, entry_id), - fbb.CreateVector(&data, 1)); - fbb.Finish(message); - } break; - case REDISMODULE_KEYTYPE_ZSET: { - // Build the flatbuffer from the set of log entries. - RAY_CHECK(RedisModule_ZsetFirstInScoreRange( - table_key, REDISMODULE_NEGATIVE_INFINITE, - REDISMODULE_POSITIVE_INFINITE, 1, 1) == REDISMODULE_OK); - std::vector> data; - for (; !RedisModule_ZsetRangeEndReached(table_key); - RedisModule_ZsetRangeNext(table_key)) { - data.push_back(RedisStringToFlatbuf( - fbb, RedisModule_ZsetRangeCurrentElement(table_key, NULL))); - } - auto message = CreateGcsTableEntry(fbb, RedisStringToFlatbuf(fbb, entry_id), - fbb.CreateVector(data)); - fbb.Finish(message); - } break; - case REDISMODULE_KEYTYPE_EMPTY: { - auto message = CreateGcsTableEntry( - fbb, RedisStringToFlatbuf(fbb, entry_id), - fbb.CreateVector( - std::vector>())); - fbb.Finish(message); - } break; - default: - RAY_LOG(FATAL) << "Invalid Redis type during lookup: " << key_type; - } -} - -/// Lookup the current value or values at a key. Returns the current value or -/// values at the key. -/// -/// This is called from a client with the command: -// -/// RAY.TABLE_LOOKUP -/// -/// \param table_prefix The prefix string for keys in this table. -/// \param pubsub_channel The pubsub channel name that notifications for -/// this key should be published to. This field is unused for lookups. -/// \param id The ID of the key to lookup. -/// \return nil if the key is empty, the current value if the key type is a -/// string, or an array of the current values if the key type is a set. -int TableLookup_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc < 4) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *prefix_str = argv[1]; - RedisModuleString *id = argv[3]; - - // Lookup the data at the key. - RedisModuleKey *table_key = - OpenPrefixedKey(ctx, prefix_str, id, REDISMODULE_READ); - if (table_key == nullptr) { - RedisModule_ReplyWithNull(ctx); - } else { - // Serialize the data to a flatbuffer to return to the client. - flatbuffers::FlatBufferBuilder fbb; - TableEntryToFlatbuf(table_key, id, fbb); - RedisModule_ReplyWithStringBuffer( - ctx, reinterpret_cast(fbb.GetBufferPointer()), - fbb.GetSize()); - } - return REDISMODULE_OK; -} - -/// Request notifications for changes to a key. Returns the current value or -/// values at the key. Notifications will be sent to the requesting client for -/// every subsequent TABLE_ADD to the key. -/// -/// This is called from a client with the command: -// -/// RAY.TABLE_REQUEST_NOTIFICATIONS -/// -/// -/// \param table_prefix The prefix string for keys in this table. -/// \param pubsub_channel The pubsub channel name that notifications for -/// this key should be published to. When publishing to a specific -/// client, the channel name should be :. -/// \param id The ID of the key to publish notifications for. -/// \param client_id The ID of the client that is being notified. -/// \return nil if the key is empty, the current value if the key type is a -/// string, or an array of the current values if the key type is a set. -int TableRequestNotifications_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 5) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *prefix_str = argv[1]; - RedisModuleString *pubsub_channel_str = argv[2]; - RedisModuleString *id = argv[3]; - RedisModuleString *client_id = argv[4]; - RedisModuleString *client_channel = - FormatPubsubChannel(ctx, pubsub_channel_str, client_id); - - // Add this client to the set of clients that should be notified when there - // are changes to the key. - RedisModuleKey *notification_key = OpenBroadcastKey( - ctx, pubsub_channel_str, id, REDISMODULE_READ | REDISMODULE_WRITE); - CHECK_ERROR(RedisModule_ZsetAdd(notification_key, 0.0, client_channel, NULL), - "ZsetAdd failed."); - - // Lookup the current value at the key. - RedisModuleKey *table_key = - OpenPrefixedKey(ctx, prefix_str, id, REDISMODULE_READ); - // Publish the current value at the key to the client that is requesting - // notifications. An empty notification will be published if the key is - // empty. - flatbuffers::FlatBufferBuilder fbb; - TableEntryToFlatbuf(table_key, id, fbb); - RedisModule_Call(ctx, "PUBLISH", "sb", client_channel, - reinterpret_cast(fbb.GetBufferPointer()), - fbb.GetSize()); - - return RedisModule_ReplyWithNull(ctx); -} - -/// Cancel notifications for changes to a key. The client will no longer -/// receive notifications for this key. This does not check if the client -/// first requested notifications before canceling them. -/// -/// This is called from a client with the command: -// -/// RAY.TABLE_CANCEL_NOTIFICATIONS -/// -/// -/// \param table_prefix The prefix string for keys in this table. -/// \param pubsub_channel The pubsub channel name that notifications for -/// this key should be published to. If publishing to a specific client, -/// then the channel name should be :. -/// \param id The ID of the key to publish notifications for. -/// \param client_id The ID of the client to cancel notifications for. -/// \return OK. -int TableCancelNotifications_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc < 5) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *pubsub_channel_str = argv[2]; - RedisModuleString *id = argv[3]; - RedisModuleString *client_id = argv[4]; - RedisModuleString *client_channel = - FormatPubsubChannel(ctx, pubsub_channel_str, client_id); - - // Remove this client from the set of clients that should be notified when - // there are changes to the key. - RedisModuleKey *notification_key = OpenBroadcastKey( - ctx, pubsub_channel_str, id, REDISMODULE_READ | REDISMODULE_WRITE); - if (RedisModule_KeyType(notification_key) != REDISMODULE_KEYTYPE_EMPTY) { - RAY_CHECK(RedisModule_ZsetRem(notification_key, client_channel, NULL) == - REDISMODULE_OK); - } - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -bool is_nil(const std::string &data) { - RAY_CHECK(data.size() == kUniqueIDSize); - const uint8_t *d = reinterpret_cast(data.data()); - for (int i = 0; i < kUniqueIDSize; ++i) { - if (d[i] != 255) { - return false; - } - } - return true; -} - -// This is a temporary redis command that will be removed once -// the GCS uses https://github.com/pcmoritz/credis. -// Be careful, this only supports Task Table payloads. -int TableTestAndUpdate_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 5) { - return RedisModule_WrongArity(ctx); - } - RedisModuleString *prefix_str = argv[1]; - RedisModuleString *id = argv[3]; - RedisModuleString *update_data = argv[4]; - - RedisModuleKey *key = OpenPrefixedKey(ctx, prefix_str, id, - REDISMODULE_READ | REDISMODULE_WRITE); - - size_t value_len = 0; - char *value_buf = RedisModule_StringDMA(key, &value_len, REDISMODULE_READ); - - size_t update_len = 0; - const char *update_buf = RedisModule_StringPtrLen(update_data, &update_len); - - auto data = flatbuffers::GetMutableRoot( - reinterpret_cast(value_buf)); - - auto update = flatbuffers::GetRoot(update_buf); - - bool do_update = static_cast(data->scheduling_state()) & - static_cast(update->test_state_bitmask()); - - if (!is_nil(update->test_scheduler_id()->str())) { - do_update = - do_update && - update->test_scheduler_id()->str() == data->scheduler_id()->str(); - } - - if (do_update) { - RAY_CHECK(data->mutate_scheduling_state(update->update_state())); - } - RAY_CHECK(data->mutate_updated(do_update)); - - int result = RedisModule_ReplyWithStringBuffer(ctx, value_buf, value_len); - - return result; -} - -/** - * Add a new entry to the object table or update an existing one. - * - * This is called from a client with the command: - * - * RAY.OBJECT_TABLE_ADD - * - * @param object_id A string representing the object ID. - * @param data_size An integer which is the object size in bytes. - * @param hash_string A string which is a hash of the object. - * @param manager A string which represents the manager ID of the plasma manager - * that has the object. - * @return OK if the operation was successful. If the same object_id is already - * present with a different hash value, the entry is still added, but - * an error with string "hash mismatch" is returned. - */ -int ObjectTableAdd_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 5) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *object_id = argv[1]; - RedisModuleString *data_size = argv[2]; - RedisModuleString *new_hash = argv[3]; - RedisModuleString *manager = argv[4]; - - long long data_size_value; - if (RedisModule_StringToLongLong(data_size, &data_size_value) != - REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "data_size must be integer"); - } - - /* Set the fields in the object info table. */ - RedisModuleKey *key; - key = OpenPrefixedKey(ctx, OBJECT_INFO_PREFIX, object_id, - REDISMODULE_READ | REDISMODULE_WRITE); - - /* Check if this object was already registered and if the hashes agree. */ - bool hash_mismatch = false; - if (RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_EMPTY) { - RedisModuleString *existing_hash; - RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "hash", &existing_hash, - NULL); - /* The existing hash may be NULL even if the key is present because a call - * to RAY.RESULT_TABLE_ADD may have already created the key. */ - if (existing_hash != NULL) { - /* Check whether the new hash value matches the old one. If not, we will - * later return the "hash mismatch" error. */ - hash_mismatch = (RedisModule_StringCompare(existing_hash, new_hash) != 0); - } - } - - RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "hash", new_hash, NULL); - RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "data_size", data_size, - NULL); - - /* Add the location in the object location table. */ - RedisModuleKey *table_key; - table_key = OpenPrefixedKey(ctx, OBJECT_LOCATION_PREFIX, object_id, - REDISMODULE_READ | REDISMODULE_WRITE); - - /* Sets are not implemented yet, so we use ZSETs instead. */ - RedisModule_ZsetAdd(table_key, 0.0, manager, NULL); - - RedisModuleString *bcast_client_str = - RedisModule_CreateString(ctx, OBJECT_BCAST, strlen(OBJECT_BCAST)); - bool success = PublishObjectNotification(ctx, bcast_client_str, object_id, - data_size, table_key); - if (!success) { - /* The publish failed somehow. */ - return RedisModule_ReplyWithError(ctx, "PUBLISH BCAST unsuccessful"); - } - - /* Get the zset of clients that requested a notification about the - * availability of this object. */ - RedisModuleKey *object_notification_key = - OpenPrefixedKey(ctx, OBJECT_NOTIFICATION_PREFIX, object_id, - REDISMODULE_READ | REDISMODULE_WRITE); - /* If the zset exists, initialize the key to iterate over the zset. */ - if (RedisModule_KeyType(object_notification_key) != - REDISMODULE_KEYTYPE_EMPTY) { - CHECK_ERROR(RedisModule_ZsetFirstInScoreRange( - object_notification_key, REDISMODULE_NEGATIVE_INFINITE, - REDISMODULE_POSITIVE_INFINITE, 1, 1), - "Unable to initialize zset iterator"); - /* Iterate over the list of clients that requested notifiations about the - * availability of this object, and publish notifications to their object - * notification channels. */ - - do { - RedisModuleString *client_id = - RedisModule_ZsetRangeCurrentElement(object_notification_key, NULL); - /* TODO(rkn): Some computation could be saved by batching the string - * constructions in the multiple calls to PublishObjectNotification - * together. */ - bool success = PublishObjectNotification(ctx, client_id, object_id, - data_size, table_key); - if (!success) { - /* The publish failed somehow. */ - return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful"); - } - } while (RedisModule_ZsetRangeNext(object_notification_key)); - /* Now that the clients have been notified, remove the zset of clients - * waiting for notifications. */ - CHECK_ERROR(RedisModule_DeleteKey(object_notification_key), - "Unable to delete zset key."); - } - - if (hash_mismatch) { - return RedisModule_ReplyWithError(ctx, "hash mismatch"); - } else { - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; - } -} - -/** - * Remove a manager from a location entry in the object table. - * - * This is called from a client with the command: - * - * RAY.OBJECT_TABLE_REMOVE - * - * @param object_id A string representing the object ID. - * @param manager A string which represents the manager ID of the plasma manager - * to remove. - * @return OK if the operation was successful or an error with string - * "object not found" if the entry for the object_id doesn't exist. The - * operation is counted as a success if the manager was already not in - * the entry. - */ -int ObjectTableRemove_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 3) { - return RedisModule_WrongArity(ctx); - } - - RedisModuleString *object_id = argv[1]; - RedisModuleString *manager = argv[2]; - - /* Remove the location from the object location table. */ - RedisModuleKey *table_key; - table_key = OpenPrefixedKey(ctx, OBJECT_LOCATION_PREFIX, object_id, - REDISMODULE_READ | REDISMODULE_WRITE); - if (RedisModule_KeyType(table_key) == REDISMODULE_KEYTYPE_EMPTY) { - return RedisModule_ReplyWithError(ctx, "object not found"); - } - - RedisModule_ZsetRem(table_key, manager, NULL); - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -/** - * Request notifications about the presence of some object IDs. This command - * takes a list of object IDs. For each object ID, the reply will be the list - * of plasma managers that contain the object. If the list of plasma managers - * is currently nonempty, then the reply will happen immediately. Else, the - * reply will come later, on the first invocation of `RAY.OBJECT_TABLE_ADD` - * following this call. - * - * This is called from a client with the command: - * - * RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS - * ... - * - * @param client_id The ID of the client that is requesting the notifications. - * @param object_id(n) The ID of the nth object ID that is passed to this - * command. This command can take any number of object IDs. - * @return OK if the operation was successful. - */ -int ObjectTableRequestNotifications_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc < 3) { - return RedisModule_WrongArity(ctx); - } - - /* The first argument is the client ID. The other arguments are object IDs. */ - RedisModuleString *client_id = argv[1]; - - /* Loop over the object ID arguments to this command. */ - for (int i = 2; i < argc; ++i) { - RedisModuleString *object_id = argv[i]; - RedisModuleKey *key = OpenPrefixedKey(ctx, OBJECT_LOCATION_PREFIX, - object_id, REDISMODULE_READ); - if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY || - RedisModule_ValueLength(key) == 0) { - /* This object ID is currently not present, so make a note that this - * client should be notified when this object ID becomes available. */ - RedisModuleKey *object_notification_key = - OpenPrefixedKey(ctx, OBJECT_NOTIFICATION_PREFIX, object_id, - REDISMODULE_READ | REDISMODULE_WRITE); - /* Add this client to the list of clients that will be notified when this - * object becomes available. */ - CHECK_ERROR( - RedisModule_ZsetAdd(object_notification_key, 0.0, client_id, NULL), - "ZsetAdd failed."); - } else { - /* Publish a notification to the client's object notification channel. */ - /* Extract the data_size first. */ - RedisModuleKey *object_info_key; - object_info_key = - OpenPrefixedKey(ctx, OBJECT_INFO_PREFIX, object_id, REDISMODULE_READ); - if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) { - return RedisModule_ReplyWithError(ctx, "requested object not found"); - } - RedisModuleString *existing_data_size; - RedisModule_HashGet(object_info_key, REDISMODULE_HASH_CFIELDS, - "data_size", &existing_data_size, NULL); - if (existing_data_size == NULL) { - return RedisModule_ReplyWithError(ctx, - "no data_size field in object info"); - } - - bool success = PublishObjectNotification(ctx, client_id, object_id, - existing_data_size, key); - if (!success) { - /* The publish failed somehow. */ - return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful"); - } - } - } - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - return REDISMODULE_OK; -} - -int ObjectInfoSubscribe_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); - return REDISMODULE_OK; -} - -/** - * Add a new entry to the result table or update an existing one. - * - * This is called from a client with the command: - * - * RAY.RESULT_TABLE_ADD - * - * @param object_id A string representing the object ID. - * @param task_id A string representing the task ID of the task that produced - * the object. - * @param is_put An integer that is 1 if the object was created through ray.put - * and 0 if created by return value. - * @return OK if the operation was successful. - */ -int ResultTableAdd_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 4) { - return RedisModule_WrongArity(ctx); - } - - /* Set the task ID under field "task" in the object info table. */ - RedisModuleString *object_id = argv[1]; - RedisModuleString *task_id = argv[2]; - RedisModuleString *is_put = argv[3]; - - /* Check to make sure the is_put field was a 0 or a 1. */ - long long is_put_integer; - if ((RedisModule_StringToLongLong(is_put, &is_put_integer) != - REDISMODULE_OK) || - (is_put_integer != 0 && is_put_integer != 1)) { - return RedisModule_ReplyWithError( - ctx, "The is_put field must be either a 0 or a 1."); - } - - RedisModuleKey *key; - key = OpenPrefixedKey(ctx, OBJECT_INFO_PREFIX, object_id, REDISMODULE_WRITE); - RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "task", task_id, "is_put", - is_put, NULL); - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - - return REDISMODULE_OK; -} - -/** - * Reply with information about a task ID. This is used by - * RAY.RESULT_TABLE_LOOKUP and RAY.TASK_TABLE_GET. - * - * @param ctx The Redis context. - * @param task_id The task ID of the task to reply about. - * @param updated A boolean representing whether the task was updated during - * this operation. This field is only used for - * RAY.TASK_TABLE_TEST_AND_UPDATE operations. - * @return NIL if the task ID is not in the task table. An error if the task ID - * is in the task table but the appropriate fields are not there, and - * an array of the task scheduling state, the local scheduler ID, and - * the task spec for the task otherwise. - */ -int ReplyWithTask(RedisModuleCtx *ctx, - RedisModuleString *task_id, - bool updated) { - RedisModuleKey *key = - OpenPrefixedKey(ctx, TASK_PREFIX, task_id, REDISMODULE_READ); - - if (RedisModule_KeyType(key) != REDISMODULE_KEYTYPE_EMPTY) { - /* If the key exists, look up the fields and return them in an array. */ - RedisModuleString *state = NULL; - RedisModuleString *local_scheduler_id = NULL; - RedisModuleString *execution_dependencies = NULL; - RedisModuleString *task_spec = NULL; - RedisModuleString *spillback_count = NULL; - RedisModule_HashGet( - key, REDISMODULE_HASH_CFIELDS, "state", &state, "local_scheduler_id", - &local_scheduler_id, "execution_dependencies", &execution_dependencies, - "TaskSpec", &task_spec, "spillback_count", &spillback_count, NULL); - if (state == NULL || local_scheduler_id == NULL || - execution_dependencies == NULL || task_spec == NULL || - spillback_count == NULL) { - /* We must have either all fields or no fields. */ - return RedisModule_ReplyWithError( - ctx, "Missing fields in the task table entry"); - } - - long long state_integer; - long long spillback_count_val; - if ((RedisModule_StringToLongLong(state, &state_integer) != - REDISMODULE_OK) || - (state_integer < 0) || - (RedisModule_StringToLongLong(spillback_count, &spillback_count_val) != - REDISMODULE_OK) || - (spillback_count_val < 0)) { - return RedisModule_ReplyWithError( - ctx, "Found invalid scheduling state or spillback count."); - } - - flatbuffers::FlatBufferBuilder fbb; - auto message = CreateTaskReply( - fbb, RedisStringToFlatbuf(fbb, task_id), state_integer, - RedisStringToFlatbuf(fbb, local_scheduler_id), - RedisStringToFlatbuf(fbb, execution_dependencies), - RedisStringToFlatbuf(fbb, task_spec), spillback_count_val, updated); - fbb.Finish(message); - - RedisModuleString *reply = RedisModule_CreateString( - ctx, (char *) fbb.GetBufferPointer(), fbb.GetSize()); - RedisModule_ReplyWithString(ctx, reply); - } else { - /* If the key does not exist, return nil. */ - RedisModule_ReplyWithNull(ctx); - } - - return REDISMODULE_OK; -} - -/** - * Lookup an entry in the result table. - * - * This is called from a client with the command: - * - * RAY.RESULT_TABLE_LOOKUP - * - * @param object_id A string representing the object ID. - * @return NIL if the object ID is not in the result table. Otherwise, this - * returns a ResultTableReply flatbuffer. - */ -int ResultTableLookup_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 2) { - return RedisModule_WrongArity(ctx); - } - - /* Get the task ID under field "task" in the object info table. */ - RedisModuleString *object_id = argv[1]; - - RedisModuleKey *key; - key = OpenPrefixedKey(ctx, OBJECT_INFO_PREFIX, object_id, REDISMODULE_READ); - - if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) { - return RedisModule_ReplyWithNull(ctx); - } - - RedisModuleString *task_id; - RedisModuleString *is_put; - RedisModuleString *data_size; - RedisModuleString *hash; - RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "task", &task_id, "is_put", - &is_put, "data_size", &data_size, "hash", &hash, NULL); - - if (task_id == NULL || is_put == NULL) { - return RedisModule_ReplyWithNull(ctx); - } - - /* Check to make sure the is_put field was a 0 or a 1. */ - long long is_put_integer; - if (RedisModule_StringToLongLong(is_put, &is_put_integer) != REDISMODULE_OK || - (is_put_integer != 0 && is_put_integer != 1)) { - return RedisModule_ReplyWithError( - ctx, "The is_put field must be either a 0 or a 1."); - } - - /* Make and return the flatbuffer reply. */ - flatbuffers::FlatBufferBuilder fbb; - long long data_size_value; - - if (data_size == NULL) { - data_size_value = -1; - } else { - RedisModule_StringToLongLong(data_size, &data_size_value); - RAY_CHECK(RedisModule_StringToLongLong(data_size, &data_size_value) == - REDISMODULE_OK); - } - - flatbuffers::Offset hash_str; - if (hash == NULL) { - hash_str = fbb.CreateString("", strlen("")); - } else { - hash_str = RedisStringToFlatbuf(fbb, hash); - } - - flatbuffers::Offset message = - CreateResultTableReply(fbb, RedisStringToFlatbuf(fbb, task_id), - bool(is_put_integer), data_size_value, hash_str); - - fbb.Finish(message); - RedisModuleString *reply = RedisModule_CreateString( - ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize()); - RedisModule_ReplyWithString(ctx, reply); - - return REDISMODULE_OK; -} - -int TaskTableWrite(RedisModuleCtx *ctx, - RedisModuleString *task_id, - RedisModuleString *state, - RedisModuleString *local_scheduler_id, - RedisModuleString *execution_dependencies, - RedisModuleString *spillback_count, - RedisModuleString *task_spec) { - /* Extract the scheduling state. */ - long long state_value; - if (RedisModule_StringToLongLong(state, &state_value) != REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "scheduling state must be integer"); - } - - long long spillback_count_value; - if (RedisModule_StringToLongLong(spillback_count, &spillback_count_value) != - REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "spillback count must be integer"); - } - /* Add the task to the task table. If no spec was provided, get the existing - * spec out of the task table so we can publish it. */ - RedisModuleString *existing_task_spec = NULL; - RedisModuleKey *key = - OpenPrefixedKey(ctx, TASK_PREFIX, task_id, REDISMODULE_WRITE); - if (task_spec == NULL) { - RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "state", state, - "local_scheduler_id", local_scheduler_id, - "execution_dependencies", execution_dependencies, - "spillback_count", spillback_count, NULL); - RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "TaskSpec", - &existing_task_spec, NULL); - if (existing_task_spec == NULL) { - return RedisModule_ReplyWithError( - ctx, "Cannot update a task that doesn't exist yet"); - } - } else { - RedisModule_HashSet( - key, REDISMODULE_HASH_CFIELDS, "state", state, "local_scheduler_id", - local_scheduler_id, "execution_dependencies", execution_dependencies, - "TaskSpec", task_spec, "spillback_count", spillback_count, NULL); - } - - if (static_cast(state_value) == TaskStatus::WAITING || - static_cast(state_value) == TaskStatus::SCHEDULED) { - /* Build the PUBLISH topic and message for task table subscribers. The - * topic is a string in the format - * "TASK_PREFIX::". The message is a serialized - * SubscribeToTasksReply flatbuffer object. */ - RedisModuleString *publish_topic = RedisString_Format( - ctx, "%s%S:%S", TASK_PREFIX, local_scheduler_id, state); - - /* Construct the flatbuffers object for the payload. */ - flatbuffers::FlatBufferBuilder fbb; - /* Use the old task spec if the current one is NULL. */ - RedisModuleString *task_spec_to_use; - if (task_spec != NULL) { - task_spec_to_use = task_spec; - } else { - task_spec_to_use = existing_task_spec; - } - /* Create the flatbuffers message. */ - auto message = CreateTaskReply( - fbb, RedisStringToFlatbuf(fbb, task_id), state_value, - RedisStringToFlatbuf(fbb, local_scheduler_id), - RedisStringToFlatbuf(fbb, execution_dependencies), - RedisStringToFlatbuf(fbb, task_spec_to_use), spillback_count_value, - true); // The updated field is not used. - fbb.Finish(message); - - RedisModuleString *publish_message = RedisModule_CreateString( - ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize()); - - RedisModuleCallReply *reply = - RedisModule_Call(ctx, "PUBLISH", "ss", publish_topic, publish_message); - - /* See how many clients received this publish. */ - long long num_clients = RedisModule_CallReplyInteger(reply); - RAY_CHECK(num_clients <= 1) << "Published to " << num_clients - << " clients."; - - if (reply == NULL) { - return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful"); - } - - if (num_clients == 0) { - /* This reply will be received by redis_task_table_update_callback or - * redis_task_table_add_task_callback in redis.cc, which will then reissue - * the command. */ - return RedisModule_ReplyWithError(ctx, - "No subscribers received message."); - } - } - - RedisModule_ReplyWithSimpleString(ctx, "OK"); - - return REDISMODULE_OK; -} - -/** - * Add a new entry to the task table. This will overwrite any existing entry - * with the same task ID. - * - * This is called from a client with the command: - * - * RAY.TASK_TABLE_ADD - * - * - * @param task_id A string that is the ID of the task. - * @param state A string that is the current scheduling state (a - * scheduling_state enum instance). - * @param local_scheduler_id A string that is the ray client ID of the - * associated local scheduler, if any. - * @param execution_dependencies A string that is the list of execution - * dependencies. - * @param task_spec A string that is the specification of the task, which can - * be cast to a `task_spec`. - * @return OK if the operation was successful. - */ -int TaskTableAddTask_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 7) { - return RedisModule_WrongArity(ctx); - } - - return TaskTableWrite(ctx, argv[1], argv[2], argv[3], argv[4], argv[5], - argv[6]); -} - -/** - * Update an entry in the task table. This does not update the task - * specification in the table. - * - * This is called from a client with the command: - * - * RAY.TASK_TABLE_UPDATE - * - * - * @param task_id A string that is the ID of the task. - * @param state A string that is the current scheduling state (a - * scheduling_state enum instance). - * @param ray_client_id A string that is the ray client ID of the associated - * local scheduler, if any. - * @param execution_dependencies A string that is the list of execution - * dependencies. - * @return OK if the operation was successful. - */ -int TaskTableUpdate_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 6) { - return RedisModule_WrongArity(ctx); - } - - return TaskTableWrite(ctx, argv[1], argv[2], argv[3], argv[4], argv[5], NULL); -} - -/** - * Test and update an entry in the task table if the current value matches the - * test value bitmask. This does not update the task specification in the - * table. - * - * This is called from a client with the command: - * - * RAY.TASK_TABLE_TEST_AND_UPDATE - * - * - * @param task_id A string that is the ID of the task. - * @param test_state_bitmask A string that is the test bitmask for the - * scheduling state. The update happens if and only if the current - * scheduling state AND-ed with the bitmask is greater than 0. - * @param state A string that is the scheduling state (a scheduling_state enum - * instance) to update the task entry with. - * @param ray_client_id A string that is the ray client ID of the associated - * local scheduler, if any, to update the task entry with. - * @param test_local_scheduler_id A string to test the local scheduler ID. If - * provided, and if the current local scheduler ID does not match it, - * then the update does not happen. - * @return Returns the task entry as a TaskReply. The reply will reflect the - * update, if it happened. - */ -int TaskTableTestAndUpdate_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc < 5 || argc > 6) { - return RedisModule_WrongArity(ctx); - } - /* If a sixth argument was provided, then we should also test the current - * local scheduler ID. */ - bool test_local_scheduler = (argc == 6); - - RedisModuleString *task_id = argv[1]; - RedisModuleString *test_state = argv[2]; - RedisModuleString *update_state = argv[3]; - RedisModuleString *local_scheduler_id = argv[4]; - - RedisModuleKey *key = OpenPrefixedKey(ctx, TASK_PREFIX, task_id, - REDISMODULE_READ | REDISMODULE_WRITE); - if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_EMPTY) { - return RedisModule_ReplyWithNull(ctx); - } - - /* If the key exists, look up the fields and return them in an array. */ - RedisModuleString *current_state = NULL; - RedisModuleString *current_local_scheduler_id = NULL; - RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "state", ¤t_state, - "local_scheduler_id", ¤t_local_scheduler_id, NULL); - - long long current_state_integer; - if (RedisModule_StringToLongLong(current_state, ¤t_state_integer) != - REDISMODULE_OK) { - return RedisModule_ReplyWithError(ctx, "current_state must be integer"); - } - - if (current_state_integer < 0) { - return RedisModule_ReplyWithError(ctx, "Found invalid scheduling state."); - } - long long test_state_bitmask; - int status = RedisModule_StringToLongLong(test_state, &test_state_bitmask); - if (status != REDISMODULE_OK) { - return RedisModule_ReplyWithError( - ctx, "Invalid test value for scheduling state"); - } - - bool update = false; - if (current_state_integer & test_state_bitmask) { - if (test_local_scheduler) { - /* A test local scheduler ID was provided. Test whether it is equal to - * the current local scheduler ID before performing the update. */ - RedisModuleString *test_local_scheduler_id = argv[5]; - if (RedisModule_StringCompare(current_local_scheduler_id, - test_local_scheduler_id) == 0) { - /* If the current local scheduler ID does matches the test ID, then - * perform the update. */ - update = true; - } - } else { - /* No test local scheduler ID was provided. Perform the update. */ - update = true; - } - } - - /* If the scheduling state and local scheduler ID tests passed, then perform - * the update. */ - if (update) { - RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "state", update_state, - "local_scheduler_id", local_scheduler_id, NULL); - } - - /* Construct a reply by getting the task from the task ID. */ - return ReplyWithTask(ctx, task_id, update); -} - -/** - * Get an entry from the task table. - * - * This is called from a client with the command: - * - * RAY.TASK_TABLE_GET - * - * @param task_id A string of the task ID to look up. - * @return An array of strings representing the task fields in the following - * order: 1) (integer) scheduling state 2) (string) associated local - * scheduler ID, if any 3) (string) the task specification, which can be - * cast to a task_spec. If the task ID is not in the table, returns nil. - */ -int TaskTableGet_RedisCommand(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - RedisModule_AutoMemory(ctx); - - if (argc != 2) { - return RedisModule_WrongArity(ctx); - } - - /* Construct a reply by getting the task from the task ID. */ - return ReplyWithTask(ctx, argv[1], false); -} - -extern "C" { - -/* This function must be present on each Redis module. It is used in order to - * register the commands into the Redis server. */ -int RedisModule_OnLoad(RedisModuleCtx *ctx, - RedisModuleString **argv, - int argc) { - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); - - if (RedisModule_Init(ctx, "ray", 1, REDISMODULE_APIVER_1) == - REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.connect", Connect_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.disconnect", Disconnect_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_add", TableAdd_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_append", - TableAppend_RedisCommand, "write", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_lookup", - TableLookup_RedisCommand, "readonly", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_request_notifications", - TableRequestNotifications_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_cancel_notifications", - TableCancelNotifications_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.table_test_and_update", - TableTestAndUpdate_RedisCommand, "write", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.object_table_lookup", - ObjectTableLookup_RedisCommand, "readonly", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.object_table_add", - ObjectTableAdd_RedisCommand, "write pubsub", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.object_table_remove", - ObjectTableRemove_RedisCommand, "write", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.object_table_request_notifications", - ObjectTableRequestNotifications_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.object_info_subscribe", - ObjectInfoSubscribe_RedisCommand, "pubsub", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.result_table_add", - ResultTableAdd_RedisCommand, "write", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.result_table_lookup", - ResultTableLookup_RedisCommand, "readonly", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.task_table_add", - TaskTableAddTask_RedisCommand, "write pubsub", - 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.task_table_update", - TaskTableUpdate_RedisCommand, "write pubsub", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.task_table_test_and_update", - TaskTableTestAndUpdate_RedisCommand, - "write pubsub", 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - - if (RedisModule_CreateCommand(ctx, "ray.task_table_get", - TaskTableGet_RedisCommand, "readonly", 0, 0, - 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - -#if RAY_USE_NEW_GCS - // Chain-enabled commands that depend on ray-project/credis. - if (RedisModule_CreateCommand(ctx, "ray.chain.table_add", - ChainTableAdd_RedisCommand, "write pubsub", 0, - 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } - if (RedisModule_CreateCommand(ctx, "ray.chain.table_append", - ChainTableAppend_RedisCommand, "write pubsub", - 0, 0, 0) == REDISMODULE_ERR) { - return REDISMODULE_ERR; - } -#endif - - return REDISMODULE_OK; -} - -} /* extern "C" */ diff --git a/src/common/shims/windows/getopt.c b/src/common/shims/windows/getopt.c deleted file mode 100644 index d9c4ae583..000000000 --- a/src/common/shims/windows/getopt.c +++ /dev/null @@ -1,69 +0,0 @@ -/* http://stackoverflow.com/a/17195644/541686 */ - -#include -#include - -int opterr = 1, /* if error message should be printed */ - optind = 1, /* index into parent argv vector */ - optopt, /* character checked for validity */ - optreset; /* reset getopt */ -char *optarg; /* argument associated with option */ - -#define BADCH (int) '?' -#define BADARG (int) ':' -#define EMSG "" - -/* -* getopt -- -* Parse argc/argv argument vector. -*/ -int getopt(int nargc, char *const nargv[], const char *ostr) { - static char *place = EMSG; /* option letter processing */ - const char *oli; /* option letter list index */ - - if (optreset || !*place) { /* update scanning pointer */ - optreset = 0; - if (optind >= nargc || *(place = nargv[optind]) != '-') { - place = EMSG; - return (-1); - } - if (place[1] && *++place == '-') { /* found "--" */ - ++optind; - place = EMSG; - return (-1); - } - } /* option letter okay? */ - if ((optopt = (int) *place++) == (int) ':' || !(oli = strchr(ostr, optopt))) { - /* - * if the user didn't specify '-' as an option, - * assume it means -1. - */ - if (optopt == (int) '-') - return (-1); - if (!*place) - ++optind; - if (opterr && *ostr != ':') - (void) printf("illegal option -- %c\n", optopt); - return (BADCH); - } - if (*++oli != ':') { /* don't need argument */ - optarg = NULL; - if (!*place) - ++optind; - } else { /* need an argument */ - if (*place) /* no white space */ - optarg = place; - else if (nargc <= ++optind) { /* no arg */ - place = EMSG; - if (*ostr == ':') - return (BADARG); - if (opterr) - (void) printf("option requires an argument -- %c\n", optopt); - return (BADCH); - } else /* white space */ - optarg = nargv[optind]; - place = EMSG; - ++optind; - } - return (optopt); /* dump back option letter */ -} diff --git a/src/common/shims/windows/getopt.h b/src/common/shims/windows/getopt.h deleted file mode 100644 index 1870fb87f..000000000 --- a/src/common/shims/windows/getopt.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef GETOPT_H -#define GETOPT_H - -#endif /* GETOPT_H */ diff --git a/src/common/shims/windows/msg.c b/src/common/shims/windows/msg.c deleted file mode 100644 index 5142c1aad..000000000 --- a/src/common/shims/windows/msg.c +++ /dev/null @@ -1,208 +0,0 @@ -#include - -int socketpair(int domain, int type, int protocol, int sv[2]) { - if ((domain != AF_UNIX && domain != AF_INET) || type != SOCK_STREAM) { - return INVALID_SOCKET; - } - SOCKET sockets[2]; - int r = dumb_socketpair(sockets); - sv[0] = (int) sockets[0]; - sv[1] = (int) sockets[1]; - return r; -} - -#pragma comment(lib, "IPHlpAPI.lib") - -struct _MIB_TCPROW2 { - DWORD dwState, dwLocalAddr, dwLocalPort, dwRemoteAddr, dwRemotePort, - dwOwningPid; - enum _TCP_CONNECTION_OFFLOAD_STATE dwOffloadState; -}; - -struct _MIB_TCPTABLE2 { - DWORD dwNumEntries; - struct _MIB_TCPROW2 table[1]; -}; - -DECLSPEC_IMPORT ULONG WINAPI GetTcpTable2(struct _MIB_TCPTABLE2 *TcpTable, - PULONG SizePointer, - BOOL Order); - -static DWORD getsockpid(SOCKET client) { - /* http://stackoverflow.com/a/25431340 */ - DWORD pid = 0; - - struct sockaddr_in Server = {0}; - int ServerSize = sizeof(Server); - - struct sockaddr_in Client = {0}; - int ClientSize = sizeof(Client); - - if ((getsockname(client, (struct sockaddr *) &Server, &ServerSize) == 0) && - (getpeername(client, (struct sockaddr *) &Client, &ClientSize) == 0)) { - struct _MIB_TCPTABLE2 *TcpTable = NULL; - ULONG TcpTableSize = 0; - ULONG result; - do { - result = GetTcpTable2(TcpTable, &TcpTableSize, TRUE); - if (result != ERROR_INSUFFICIENT_BUFFER) { - break; - } - free(TcpTable); - TcpTable = (struct _MIB_TCPTABLE2 *) malloc(TcpTableSize); - } while (TcpTable != NULL); - - if (result == NO_ERROR) { - for (DWORD dw = 0; dw < TcpTable->dwNumEntries; ++dw) { - struct _MIB_TCPROW2 *row = &(TcpTable->table[dw]); - if ((row->dwState == 5 /* MIB_TCP_STATE_ESTAB */) && - (row->dwLocalAddr == Client.sin_addr.s_addr) && - ((row->dwLocalPort & 0xFFFF) == Client.sin_port) && - (row->dwRemoteAddr == Server.sin_addr.s_addr) && - ((row->dwRemotePort & 0xFFFF) == Server.sin_port)) { - pid = row->dwOwningPid; - break; - } - } - } - - free(TcpTable); - } - - return pid; -} - -ssize_t sendmsg(int sockfd, struct msghdr *msg, int flags) { - ssize_t result = -1; - struct cmsghdr *header = CMSG_FIRSTHDR(msg); - if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) { - /* We're trying to send over a handle of some kind. - * We have to look up which process we're communicating with, - * open a handle to it, and then duplicate our handle into it. - * However, the first two steps cannot be done atomically. - * Therefore, this code HAS A RACE CONDITIONS and is therefore NOT SECURE. - * In the absense of a malicious actor, though, it is exceedingly unlikely - * that the child process closes AND that its process ID is reassigned - * to another existing process. - */ - struct msghdr const old_msg = *msg; - int *const pfd = (int *) CMSG_DATA(header); - msg->msg_control = NULL; - msg->msg_controllen = 0; - WSAPROTOCOL_INFO protocol_info = {0}; - BOOL const is_socket = !!FDAPI_GetSocketStatePtr(*pfd); - DWORD const target_pid = getsockpid(sockfd); - HANDLE target_process = NULL; - if (target_pid) { - if (!is_socket) { - /* This is a regular handle... fit it into the same struct */ - target_process = OpenProcess(PROCESS_DUP_HANDLE, FALSE, target_pid); - if (target_process) { - if (DuplicateHandle(GetCurrentProcess(), (HANDLE)(intptr_t) *pfd, - target_process, (HANDLE *) &protocol_info, 0, - TRUE, DUPLICATE_SAME_ACCESS)) { - result = 0; - } - } - } else { - /* This is a socket... */ - result = FDAPI_WSADuplicateSocket(*pfd, target_pid, &protocol_info); - } - } - if (result == 0) { - int const nbufs = msg->dwBufferCount + 1; - WSABUF *const bufs = - (struct _WSABUF *) _alloca(sizeof(*msg->lpBuffers) * nbufs); - bufs[0].buf = (char *) &protocol_info; - bufs[0].len = sizeof(protocol_info); - memcpy(&bufs[1], msg->lpBuffers, - msg->dwBufferCount * sizeof(*msg->lpBuffers)); - DWORD nb; - msg->lpBuffers = bufs; - msg->dwBufferCount = nbufs; - GUID const wsaid_WSASendMsg = { - 0xa441e712, - 0x754f, - 0x43ca, - {0x84, 0xa7, 0x0d, 0xee, 0x44, 0xcf, 0x60, 0x6d}}; - typedef INT PASCAL WSASendMsg_t( - SOCKET s, LPWSAMSG lpMsg, DWORD dwFlags, LPDWORD lpNumberOfBytesSent, - LPWSAOVERLAPPED lpOverlapped, - LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); - WSASendMsg_t *WSASendMsg = NULL; - result = FDAPI_WSAIoctl(sockfd, SIO_GET_EXTENSION_FUNCTION_POINTER, - &wsaid_WSASendMsg, sizeof(wsaid_WSASendMsg), - &WSASendMsg, sizeof(WSASendMsg), &nb, NULL, 0); - if (result == 0) { - result = (*WSASendMsg)(sockfd, msg, flags, &nb, NULL, NULL) == 0 - ? (ssize_t)(nb - sizeof(protocol_info)) - : 0; - } - } - if (result != 0 && target_process && !is_socket) { - /* we failed to send the handle, and it needs cleaning up! */ - HANDLE duplicated_back = NULL; - if (DuplicateHandle(target_process, *(HANDLE *) &protocol_info, - GetCurrentProcess(), &duplicated_back, 0, FALSE, - DUPLICATE_CLOSE_SOURCE)) { - CloseHandle(duplicated_back); - } - } - if (target_process) { - CloseHandle(target_process); - } - *msg = old_msg; - } - return result; -} - -ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags) { - int result = -1; - struct cmsghdr *header = CMSG_FIRSTHDR(msg); - if (msg->msg_controllen && - flags == 0 /* We can't send flags on Windows... */) { - struct msghdr const old_msg = *msg; - msg->msg_control = NULL; - msg->msg_controllen = 0; - WSAPROTOCOL_INFO protocol_info = {0}; - int const nbufs = msg->dwBufferCount + 1; - WSABUF *const bufs = - (struct _WSABUF *) _alloca(sizeof(*msg->lpBuffers) * nbufs); - bufs[0].buf = (char *) &protocol_info; - bufs[0].len = sizeof(protocol_info); - memcpy(&bufs[1], msg->lpBuffers, - msg->dwBufferCount * sizeof(*msg->lpBuffers)); - typedef INT PASCAL WSARecvMsg_t( - SOCKET s, LPWSAMSG lpMsg, LPDWORD lpNumberOfBytesRecvd, - LPWSAOVERLAPPED lpOverlapped, - LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine); - WSARecvMsg_t *WSARecvMsg = NULL; - DWORD nb; - GUID const wsaid_WSARecvMsg = { - 0xf689d7c8, - 0x6f1f, - 0x436b, - {0x8a, 0x53, 0xe5, 0x4f, 0xe3, 0x51, 0xc3, 0x22}}; - result = FDAPI_WSAIoctl(sockfd, SIO_GET_EXTENSION_FUNCTION_POINTER, - &wsaid_WSARecvMsg, sizeof(wsaid_WSARecvMsg), - &WSARecvMsg, sizeof(WSARecvMsg), &nb, NULL, 0); - if (result == 0) { - result = (*WSARecvMsg)(sockfd, msg, &nb, NULL, NULL) == 0 - ? (ssize_t)(nb - sizeof(protocol_info)) - : 0; - } - if (result == 0) { - int *const pfd = (int *) CMSG_DATA(header); - if (protocol_info.iSocketType == 0 && protocol_info.iProtocol == 0) { - *pfd = *(int *) &protocol_info; - } else { - *pfd = FDAPI_WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, - FROM_PROTOCOL_INFO, &protocol_info, 0, 0); - } - header->cmsg_level = SOL_SOCKET; - header->cmsg_type = SCM_RIGHTS; - } - *msg = old_msg; - } - return result; -} diff --git a/src/common/shims/windows/netdb.h b/src/common/shims/windows/netdb.h deleted file mode 100644 index 5dace1659..000000000 --- a/src/common/shims/windows/netdb.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef NETDB_H -#define NETDB_H - -#endif /* NETDB_H */ diff --git a/src/common/shims/windows/netinet/in.h b/src/common/shims/windows/netinet/in.h deleted file mode 100644 index a60db3e05..000000000 --- a/src/common/shims/windows/netinet/in.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef IN_H -#define IN_H - -#endif /* IN_H */ diff --git a/src/common/shims/windows/poll.h b/src/common/shims/windows/poll.h deleted file mode 100644 index 058e23ade..000000000 --- a/src/common/shims/windows/poll.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef POLL_H -#define POLL_H - -#endif /* POLL_H */ diff --git a/src/common/shims/windows/socketpair.c b/src/common/shims/windows/socketpair.c deleted file mode 100644 index e9fc792c1..000000000 --- a/src/common/shims/windows/socketpair.c +++ /dev/null @@ -1,150 +0,0 @@ -/* socketpair.c -Copyright 2007, 2010 by Nathan C. Myers -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - The name of the author must not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* Changes: - * 2014-02-12: merge David Woodhouse, Ger Hobbelt improvements - * git.infradead.org/users/dwmw2/openconnect.git/commitdiff/bdeefa54 - * github.com/GerHobbelt/selectable-socketpair - * always init the socks[] to -1/INVALID_SOCKET on error, both on Win32/64 - * and UNIX/other platforms - * 2013-07-18: Change to BSD 3-clause license - * 2010-03-31: - * set addr to 127.0.0.1 because win32 getsockname does not always set it. - * 2010-02-25: - * set SO_REUSEADDR option to avoid leaking some windows resource. - * Windows System Error 10049, "Event ID 4226 TCP/IP has reached - * the security limit imposed on the number of concurrent TCP connect - * attempts." Bleah. - * 2007-04-25: - * preserve value of WSAGetLastError() on all error returns. - * 2007-04-22: (Thanks to Matthew Gregan ) - * s/EINVAL/WSAEINVAL/ fix trivial compile failure - * s/socket/WSASocket/ enable creation of sockets suitable as stdin/stdout - * of a child process. - * add argument make_overlapped - */ - -#include - -#ifdef WIN32 -#include /* socklen_t, et al (MSVC20xx) */ -#include -#include -#else -#ifdef _WIN32 -#include -#include -#endif -#include -#include -#include -#endif - -#ifdef WIN32 - -/* dumb_socketpair: - * If make_overlapped is nonzero, both sockets created will be usable for - * "overlapped" operations via WSASend etc. If make_overlapped is zero, - * socks[0] (only) will be usable with regular ReadFile etc., and thus - * suitable for use as stdin or stdout of a child process. Note that the - * sockets must be closed with closesocket() regardless. - */ - -int dumb_socketpair(SOCKET socks[2]) { - union { - struct sockaddr_in inaddr; - struct sockaddr addr; - } a; - SOCKET listener; - int e; - socklen_t addrlen = sizeof(a.inaddr); - int reuse = 1; - - if (socks == 0) { - return SOCKET_ERROR; - } - socks[0] = socks[1] = -1; - - listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (listener == -1) - return SOCKET_ERROR; - - memset(&a, 0, sizeof(a)); - a.inaddr.sin_family = AF_INET; - a.inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - a.inaddr.sin_port = 0; - - for (;;) { - if (setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, - (socklen_t) sizeof(reuse)) == -1) - break; - if (bind(listener, &a.addr, sizeof(a.inaddr)) == SOCKET_ERROR) - break; - - memset(&a, 0, sizeof(a)); - if (getsockname(listener, &a.addr, &addrlen) == SOCKET_ERROR) - break; - // win32 getsockname may only set the port number, p=0.0005. - // ( http://msdn.microsoft.com/library/ms738543.aspx ): - a.inaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - a.inaddr.sin_family = AF_INET; - - if (listen(listener, 1) == SOCKET_ERROR) - break; - - socks[0] = FDAPI_WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, 0); - if (socks[0] == -1) - break; - if (connect(socks[0], &a.addr, sizeof(a.inaddr)) == SOCKET_ERROR) - break; - - socks[1] = accept(listener, NULL, NULL); - if (socks[1] == -1) - break; - - FDAPI_close(listener); - return 0; - } - - FDAPI_close(listener); - FDAPI_close(socks[0]); - FDAPI_close(socks[1]); - socks[0] = socks[1] = -1; - return SOCKET_ERROR; -} -#else -int dumb_socketpair(int socks[2], int dummy) { - if (socks == 0) { - errno = EINVAL; - return -1; - } - dummy = socketpair(AF_LOCAL, SOCK_STREAM, 0, socks); - if (dummy) - socks[0] = socks[1] = -1; - return dummy; -} -#endif diff --git a/src/common/shims/windows/strings.h b/src/common/shims/windows/strings.h deleted file mode 100644 index e264061c4..000000000 --- a/src/common/shims/windows/strings.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef STRINGS_H -#define STRINGS_H - -#endif /* STRINGS_H */ diff --git a/src/common/shims/windows/sys/ioctl.h b/src/common/shims/windows/sys/ioctl.h deleted file mode 100644 index 00f7a55ed..000000000 --- a/src/common/shims/windows/sys/ioctl.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef IOCTL_H -#define IOCTL_H - -#endif /* IOCTL_H */ diff --git a/src/common/shims/windows/sys/mman.h b/src/common/shims/windows/sys/mman.h deleted file mode 100644 index a12df75fc..000000000 --- a/src/common/shims/windows/sys/mman.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef MMAN_H -#define MMAN_H - -#include - -#define MAP_SHARED 0x0010 /* share changes */ -#define MAP_FAILED ((void *) -1) -#define PROT_READ 0x04 /* pages can be read */ -#define PROT_WRITE 0x02 /* pages can be written */ -#define PROT_EXEC 0x01 /* pages can be executed */ - -static void *mmap(void *addr, - size_t len, - int prot, - int flags, - int fd, - off_t off) { - void *result = (void *) (-1); - if (!addr && prot == MAP_SHARED) { - /* HACK: we're assuming handle sizes can't exceed 32 bits, which is wrong... - * but works for now. */ - void *ptr = MapViewOfFile((HANDLE)(intptr_t) fd, FILE_MAP_ALL_ACCESS, - (DWORD)(off >> (CHAR_BIT * sizeof(DWORD))), - (DWORD) off, (SIZE_T) len); - if (ptr) { - result = ptr; - } - } - return result; -} -static int munmap(void *addr, size_t length) { - (void) length; - return UnmapViewOfFile(addr) ? 0 : -1; -} - -#endif /* MMAN_H */ diff --git a/src/common/shims/windows/sys/select.h b/src/common/shims/windows/sys/select.h deleted file mode 100644 index 8aef7950e..000000000 --- a/src/common/shims/windows/sys/select.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef SELECT_H -#define SELECT_H - -#endif /* SELECT_H */ diff --git a/src/common/shims/windows/sys/socket.h b/src/common/shims/windows/sys/socket.h deleted file mode 100644 index ba9d656bb..000000000 --- a/src/common/shims/windows/sys/socket.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef SOCKET_H -#define SOCKET_H - -typedef unsigned short sa_family_t; - -#include "../../src/Win32_Interop/Win32_FDAPI.h" -#include "../../src/Win32_Interop/Win32_APIs.h" - -#define cmsghdr _WSACMSGHDR -#undef CMSG_DATA -#define CMSG_DATA WSA_CMSG_DATA -#define CMSG_SPACE WSA_CMSG_SPACE -#define CMSG_FIRSTHDR WSA_CMSG_FIRSTHDR -#define CMSG_LEN WSA_CMSG_LEN -#define CMSG_NXTHDR WSA_CMSG_NXTHDR - -#define SCM_RIGHTS 1 - -#define iovec _WSABUF -#define iov_base buf -#define iov_len len -#define msghdr _WSAMSG -#define msg_name name -#define msg_namelen namelen -#define msg_iov lpBuffers -#define msg_iovlen dwBufferCount -#define msg_control Control.buf -#define msg_controllen Control.len -#define msg_flags dwFlags - -int dumb_socketpair(SOCKET socks[2]); -ssize_t sendmsg(int sockfd, struct msghdr *msg, int flags); -ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags); -int socketpair(int domain, int type, int protocol, int sv[2]); - -#endif /* SOCKET_H */ diff --git a/src/common/shims/windows/sys/time.h b/src/common/shims/windows/sys/time.h deleted file mode 100644 index 976342bd2..000000000 --- a/src/common/shims/windows/sys/time.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef TIME_H -#define TIME_H - -#include /* timeval */ - -int gettimeofday_highres(struct timeval *tv, struct timezone *tz); - -static int gettimeofday(struct timeval *tv, struct timezone *tz) { - return gettimeofday_highres(tv, tz); -} - -#endif /* TIME_H */ diff --git a/src/common/shims/windows/sys/un.h b/src/common/shims/windows/sys/un.h deleted file mode 100644 index 91642683f..000000000 --- a/src/common/shims/windows/sys/un.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef UN_H -#define UN_H - -#include - -struct sockaddr_un { - /** AF_UNIX. */ - sa_family_t sun_family; - /** The pathname. */ - char sun_path[108]; -}; - -#endif /* UN_H */ diff --git a/src/common/shims/windows/sys/wait.h b/src/common/shims/windows/sys/wait.h deleted file mode 100644 index 442218408..000000000 --- a/src/common/shims/windows/sys/wait.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef WAIT_H -#define WAIT_H - -#endif /* WAIT_H */ diff --git a/src/common/shims/windows/unistd.h b/src/common/shims/windows/unistd.h deleted file mode 100644 index aab25417e..000000000 --- a/src/common/shims/windows/unistd.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef UNISTD_H -#define UNISTD_H - -extern char *optarg; -extern int optind, opterr, optopt; -int getopt(int nargc, char *const nargv[], const char *ostr); - -#include "../../src/Win32_Interop/Win32_FDAPI.h" -#define close(...) FDAPI_close(__VA_ARGS__) - -#endif /* UNISTD_H */ diff --git a/src/common/state/actor_notification_table.cc b/src/common/state/actor_notification_table.cc deleted file mode 100644 index 19cd7fddd..000000000 --- a/src/common/state/actor_notification_table.cc +++ /dev/null @@ -1,47 +0,0 @@ -#include "actor_notification_table.h" - -#include "common_protocol.h" -#include "redis.h" - -void publish_actor_creation_notification(DBHandle *db_handle, - const ActorID &actor_id, - const WorkerID &driver_id, - const DBClientID &local_scheduler_id) { - // Create a flatbuffer object to serialize and publish. - flatbuffers::FlatBufferBuilder fbb; - // Create the flatbuffers message. - auto message = CreateActorCreationNotification( - fbb, to_flatbuf(fbb, actor_id), to_flatbuf(fbb, driver_id), - to_flatbuf(fbb, local_scheduler_id)); - fbb.Finish(message); - - ActorCreationNotificationData *data = - (ActorCreationNotificationData *) malloc( - sizeof(ActorCreationNotificationData) + fbb.GetSize()); - data->size = fbb.GetSize(); - memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize()); - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(data), NULL, NULL, - redis_publish_actor_creation_notification, NULL); -} - -void actor_notification_table_subscribe( - DBHandle *db_handle, - actor_notification_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry) { - ActorNotificationTableSubscribeData *sub_data = - (ActorNotificationTableSubscribeData *) malloc( - sizeof(ActorNotificationTableSubscribeData)); - sub_data->subscribe_callback = subscribe_callback; - sub_data->subscribe_context = subscribe_context; - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(sub_data), retry, NULL, - redis_actor_notification_table_subscribe, NULL); -} - -void actor_table_mark_removed(DBHandle *db_handle, ActorID actor_id) { - redis_actor_table_mark_removed(db_handle, actor_id); -} diff --git a/src/common/state/actor_notification_table.h b/src/common/state/actor_notification_table.h deleted file mode 100644 index f6aa101cd..000000000 --- a/src/common/state/actor_notification_table.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef ACTOR_NOTIFICATION_TABLE_H -#define ACTOR_NOTIFICATION_TABLE_H - -#include "task.h" -#include "db.h" -#include "table.h" - -/* - * ==== Subscribing to the actor notification table ==== - */ - -/* Callback for subscribing to the local scheduler table. */ -typedef void (*actor_notification_table_subscribe_callback)( - const ActorID &actor_id, - const WorkerID &driver_id, - const DBClientID &local_scheduler_id, - void *user_context); - -/// Publish an actor creation notification. This is published by a local -/// scheduler once it creates an actor. -/// -/// \param db_handle Database handle. -/// \param actor_id The ID of the actor that was created. -/// \param driver_id The ID of the driver that created the actor. -/// \param local_scheduler_id The ID of the local scheduler that created the -/// actor. -/// \return Void. -void publish_actor_creation_notification(DBHandle *db_handle, - const ActorID &actor_id, - const WorkerID &driver_id, - const DBClientID &local_scheduler_id); - -/// Data that is needed to publish an actor creation notification. -typedef struct { - /// The size of the flatbuffer object. - int64_t size; - /// The information to be sent. - uint8_t flatbuffer_data[0]; -} ActorCreationNotificationData; - -/** - * Register a callback to process actor notification events. - * - * @param db_handle Database handle. - * @param subscribe_callback Callback that will be called when the local - * scheduler event happens. - * @param subscribe_context Context that will be passed into the - * subscribe_callback. - * @param retry Information about retrying the request to the database. - * @return Void. - */ -void actor_notification_table_subscribe( - DBHandle *db_handle, - actor_notification_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry); - -/* Data that is needed to register local scheduler table subscribe callbacks - * with the state database. */ -typedef struct { - actor_notification_table_subscribe_callback subscribe_callback; - void *subscribe_context; -} ActorNotificationTableSubscribeData; - -/** - * Marks an actor as removed. This prevents the actor from being resurrected. - * - * @param db The database handle. - * @param actor_id The actor id to mark as removed. - * @return Void. - */ -void actor_table_mark_removed(DBHandle *db_handle, ActorID actor_id); - -#endif /* ACTOR_NOTIFICATION_TABLE_H */ diff --git a/src/common/state/db.h b/src/common/state/db.h deleted file mode 100644 index ac9960b89..000000000 --- a/src/common/state/db.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef DB_H -#define DB_H - -#include - -#include "common.h" -#include "event_loop.h" - -typedef struct DBHandle DBHandle; - -/** - * Connect to the global system store. - * - * @param db_address The hostname to use to connect to the database. - * @param db_port The port to use to connect to the database. - * @param db_shards_addresses The list of database shard IP addresses. - * @param db_shards_ports The list of database shard ports, in the same order - * as db_shards_addresses. - * @param client_type The type of this client. - * @param node_ip_address The hostname of the client that is connecting. - * @param args A vector of extra arguments strings. They should alternate - * between the name of the argument and the value of the argument. For - * examples: "port", "1234", "socket_name", "/tmp/s1". This vector should - * have an even length. - * @return This returns a handle to the database, which must be freed with - * db_disconnect after use. - */ -DBHandle *db_connect(const std::string &db_primary_address, - int db_primary_port, - const char *client_type, - const char *node_ip_address, - const std::vector &args); - -/** - * Attach global system store connection to an event loop. Callbacks from - * queries to the global system store will trigger events in the event loop. - * - * @param db The handle to the database that is connected. - * @param loop The event loop the database gets connected to. - * @param reattach Can only be true in unit tests. If true, the database is - * reattached to the loop. - * @return Void. - */ -void db_attach(DBHandle *db, event_loop *loop, bool reattach); - -/** - * Disconnect from the global system store. - * - * @param db The database connection to close and clean up. - * @return Void. - */ -void db_disconnect(DBHandle *db); - -/** - * Free the database handle. - * - * @param db The database connection to clean up. - * @return Void. - */ -void DBHandle_free(DBHandle *db); - -/** - * Returns the db client ID. - * - * @param db The handle to the database. - * @returns int The db client ID for this connection to the database. - */ -DBClientID get_db_client_id(DBHandle *db); - -#endif diff --git a/src/common/state/db_client_table.cc b/src/common/state/db_client_table.cc deleted file mode 100644 index b31e9d8c2..000000000 --- a/src/common/state/db_client_table.cc +++ /dev/null @@ -1,90 +0,0 @@ -#include "db_client_table.h" -#include "redis.h" - -void db_client_table_remove(DBHandle *db_handle, - DBClientID db_client_id, - RetryInfo *retry, - db_client_table_done_callback done_callback, - void *user_context) { - init_table_callback(db_handle, db_client_id, __func__, - new CommonCallbackData(NULL), retry, - (table_done_callback) done_callback, - redis_db_client_table_remove, user_context); -} - -void db_client_table_subscribe( - DBHandle *db_handle, - db_client_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry, - db_client_table_done_callback done_callback, - void *user_context) { - DBClientTableSubscribeData *sub_data = - (DBClientTableSubscribeData *) malloc(sizeof(DBClientTableSubscribeData)); - sub_data->subscribe_callback = subscribe_callback; - sub_data->subscribe_context = subscribe_context; - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(sub_data), retry, - (table_done_callback) done_callback, - redis_db_client_table_subscribe, user_context); -} - -const std::vector db_client_table_get_ip_addresses( - DBHandle *db_handle, - const std::vector &manager_ids) { - /* We time this function because in the past this loop has taken multiple - * seconds under stressful situations on hundreds of machines causing the - * plasma manager to die (because it went too long without sending - * heartbeats). */ - int64_t start_time = current_time_ms(); - - /* Construct the manager vector from the flatbuffers object. */ - std::vector manager_vector; - - for (auto const &manager_id : manager_ids) { - DBClient client = redis_cache_get_db_client(db_handle, manager_id); - RAY_CHECK(!client.manager_address.empty()); - if (client.is_alive) { - manager_vector.push_back(client.manager_address); - } - } - - int64_t end_time = current_time_ms(); - if (end_time - start_time > RayConfig::instance().max_time_for_loop()) { - RAY_LOG(WARNING) << "calling redis_get_cached_db_client in a loop in with " - << manager_ids.size() << " manager IDs took " - << end_time - start_time << " milliseconds."; - } - - return manager_vector; -} - -void db_client_table_update_cache_callback(DBClient *db_client, - void *user_context) { - DBHandle *db_handle = (DBHandle *) user_context; - redis_cache_set_db_client(db_handle, *db_client); -} - -void db_client_table_cache_init(DBHandle *db_handle) { - db_client_table_subscribe(db_handle, db_client_table_update_cache_callback, - db_handle, NULL, NULL, NULL); -} - -DBClient db_client_table_cache_get(DBHandle *db_handle, DBClientID client_id) { - RAY_CHECK(!client_id.is_nil()); - return redis_cache_get_db_client(db_handle, client_id); -} - -void plasma_manager_send_heartbeat(DBHandle *db_handle) { - RetryInfo heartbeat_retry; - heartbeat_retry.num_retries = 0; - heartbeat_retry.timeout = - RayConfig::instance().heartbeat_timeout_milliseconds(); - heartbeat_retry.fail_callback = NULL; - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(NULL), - (RetryInfo *) &heartbeat_retry, NULL, - redis_plasma_manager_send_heartbeat, NULL); -} diff --git a/src/common/state/db_client_table.h b/src/common/state/db_client_table.h deleted file mode 100644 index d140ba770..000000000 --- a/src/common/state/db_client_table.h +++ /dev/null @@ -1,120 +0,0 @@ -#ifndef DB_CLIENT_TABLE_H -#define DB_CLIENT_TABLE_H - -#include - -#include "db.h" -#include "table.h" - -typedef void (*db_client_table_done_callback)(DBClientID db_client_id, - void *user_context); - -/** - * Remove a client from the db clients table. - * - * @param db_handle Database handle. - * @param db_client_id The database client ID to remove. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - * - */ -void db_client_table_remove(DBHandle *db_handle, - DBClientID db_client_id, - RetryInfo *retry, - db_client_table_done_callback done_callback, - void *user_context); - -/* - * ==== Subscribing to the db client table ==== - */ - -/* An entry in the db client table. */ -typedef struct { - /** The database client ID. */ - DBClientID id; - /** The database client type. */ - std::string client_type; - /** An optional auxiliary address for the plasma manager associated with this - * database client. */ - std::string manager_address; - /** Whether or not the database client exists. If this is false for an entry, - * then it will never again be true. */ - bool is_alive; -} DBClient; - -/* Callback for subscribing to the db client table. */ -typedef void (*db_client_table_subscribe_callback)(DBClient *db_client, - void *user_context); - -/** - * Register a callback for a db client table event. - * - * @param db_handle Database handle. - * @param subscribe_callback Callback that will be called when the db client - * table is updated. - * @param subscribe_context Context that will be passed into the - * subscribe_callback. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void db_client_table_subscribe( - DBHandle *db_handle, - db_client_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry, - db_client_table_done_callback done_callback, - void *user_context); - -/* Data that is needed to register db client table subscribe callbacks with the - * state database. */ -typedef struct { - db_client_table_subscribe_callback subscribe_callback; - void *subscribe_context; -} DBClientTableSubscribeData; - -const std::vector db_client_table_get_ip_addresses( - DBHandle *db, - const std::vector &manager_ids); - -/** - * Initialize the db client cache. The cache is updated with each notification - * from the db client table. - * - * @param db_handle Database handle. - * @return Void. - */ -void db_client_table_cache_init(DBHandle *db_handle); - -/** - * Get a db client from the cache. If the requested client is not there, - * request the latest entry from the db client table. - * - * @param db_handle Database handle. - * @param client_id The ID of the client to look up in the cache. - * @return The database client in the cache. - */ -DBClient db_client_table_cache_get(DBHandle *db_handle, DBClientID client_id); - -/* - * ==== Plasma manager heartbeats ==== - */ - -/** - * Start sending heartbeats to the plasma_managers channel. Each - * heartbeat contains this database client's ID. Heartbeats can be subscribed - * to through the plasma_managers channel. Once called, this "retries" the - * heartbeat operation forever, every heartbeat_timeout_milliseconds - * milliseconds. - * - * @param db_handle Database handle. - * @return Void. - */ -void plasma_manager_send_heartbeat(DBHandle *db_handle); - -#endif /* DB_CLIENT_TABLE_H */ diff --git a/src/common/state/driver_table.cc b/src/common/state/driver_table.cc deleted file mode 100644 index b8732e986..000000000 --- a/src/common/state/driver_table.cc +++ /dev/null @@ -1,23 +0,0 @@ -#include "driver_table.h" -#include "redis.h" - -void driver_table_subscribe(DBHandle *db_handle, - driver_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry) { - DriverTableSubscribeData *sub_data = - (DriverTableSubscribeData *) malloc(sizeof(DriverTableSubscribeData)); - sub_data->subscribe_callback = subscribe_callback; - sub_data->subscribe_context = subscribe_context; - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(sub_data), retry, NULL, - redis_driver_table_subscribe, NULL); -} - -void driver_table_send_driver_death(DBHandle *db_handle, - WorkerID driver_id, - RetryInfo *retry) { - init_table_callback(db_handle, driver_id, __func__, - new CommonCallbackData(NULL), retry, NULL, - redis_driver_table_send_driver_death, NULL); -} diff --git a/src/common/state/driver_table.h b/src/common/state/driver_table.h deleted file mode 100644 index c8c6a6c32..000000000 --- a/src/common/state/driver_table.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef DRIVER_TABLE_H -#define DRIVER_TABLE_H - -#include "db.h" -#include "table.h" -#include "task.h" - -/* - * ==== Subscribing to the driver table ==== - */ - -/* Callback for subscribing to the driver table. */ -typedef void (*driver_table_subscribe_callback)(WorkerID driver_id, - void *user_context); - -/** - * Register a callback for a driver table event. - * - * @param db_handle Database handle. - * @param subscribe_callback Callback that will be called when the driver event - * happens. - * @param subscribe_context Context that will be passed into the - * subscribe_callback. - * @param retry Information about retrying the request to the database. - * @return Void. - */ -void driver_table_subscribe(DBHandle *db_handle, - driver_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry); - -/* Data that is needed to register driver table subscribe callbacks with the - * state database. */ -typedef struct { - driver_table_subscribe_callback subscribe_callback; - void *subscribe_context; -} DriverTableSubscribeData; - -/** - * Send driver death update to all subscribers. - * - * @param db_handle Database handle. - * @param driver_id The ID of the driver that died. - * @param retry Information about retrying the request to the database. - */ -void driver_table_send_driver_death(DBHandle *db_handle, - WorkerID driver_id, - RetryInfo *retry); - -#endif /* DRIVER_TABLE_H */ diff --git a/src/common/state/error_table.cc b/src/common/state/error_table.cc deleted file mode 100644 index d0fd9bdff..000000000 --- a/src/common/state/error_table.cc +++ /dev/null @@ -1,24 +0,0 @@ -#include "error_table.h" -#include "redis.h" - -const char *error_types[] = {"object_hash_mismatch", "put_reconstruction", - "worker_died", "actor_not_created"}; - -void push_error(DBHandle *db_handle, - DBClientID driver_id, - ErrorIndex error_type, - const std::string &error_message) { - int64_t message_size = error_message.size(); - - /* Allocate a struct to hold the error information. */ - ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + message_size); - info->driver_id = driver_id; - info->error_type = error_type; - info->error_key = UniqueID::from_random(); - info->size = message_size; - memcpy(info->error_message, error_message.data(), message_size); - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(info), NULL, NULL, - redis_push_error, NULL); -} diff --git a/src/common/state/error_table.h b/src/common/state/error_table.h deleted file mode 100644 index 908d7f4d0..000000000 --- a/src/common/state/error_table.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef ERROR_TABLE_H -#define ERROR_TABLE_H - -#include "db.h" -#include "table.h" - -/// An ErrorIndex may be used as an index into error_types. -enum class ErrorIndex : int32_t { - /// An object was added with a different hash from the existing one. - OBJECT_HASH_MISMATCH = 0, - /// An object that was created through a ray.put is lost. - PUT_RECONSTRUCTION, - /// A worker died or was killed while executing a task. - WORKER_DIED, - /// An actor hasn't been created for a while. - ACTOR_NOT_CREATED, - /// The total number of error types. - MAX -}; - -/// Data that is needed to push an error. -typedef struct { - /// The ID of the driver to push the error to. - DBClientID driver_id; - /// An index into the error_types array indicating the type of the error. - ErrorIndex error_type; - /// The key to use for the error message in Redis. - UniqueID error_key; - /// The length of the error message. - int64_t size; - /// The error message. - uint8_t error_message[0]; -} ErrorInfo; - -extern const char *error_types[]; - -/// Push an error to the given Python driver. -/// -/// \param db_handle Database handle. -/// \param driver_id The ID of the Python driver to push the error to. -/// \param error_type An index specifying the type of the error. This should -/// be a value from the ErrorIndex enum. -/// \param error_message The error message to print. -/// \return Void. -void push_error(DBHandle *db_handle, - DBClientID driver_id, - ErrorIndex error_type, - const std::string &error_message); - -#endif diff --git a/src/common/state/local_scheduler_table.cc b/src/common/state/local_scheduler_table.cc deleted file mode 100644 index 075d52102..000000000 --- a/src/common/state/local_scheduler_table.cc +++ /dev/null @@ -1,48 +0,0 @@ -#include "local_scheduler_table.h" - -#include "common_protocol.h" -#include "redis.h" - -void local_scheduler_table_subscribe( - DBHandle *db_handle, - local_scheduler_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry) { - LocalSchedulerTableSubscribeData *sub_data = - (LocalSchedulerTableSubscribeData *) malloc( - sizeof(LocalSchedulerTableSubscribeData)); - sub_data->subscribe_callback = subscribe_callback; - sub_data->subscribe_context = subscribe_context; - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(sub_data), retry, NULL, - redis_local_scheduler_table_subscribe, NULL); -} - -void local_scheduler_table_send_info(DBHandle *db_handle, - LocalSchedulerInfo *info, - RetryInfo *retry) { - /* Create a flatbuffer object to serialize and publish. */ - flatbuffers::FlatBufferBuilder fbb; - /* Create the flatbuffers message. */ - auto message = CreateLocalSchedulerInfoMessage( - fbb, to_flatbuf(fbb, db_handle->client), info->total_num_workers, - info->task_queue_length, info->available_workers, - map_to_flatbuf(fbb, info->static_resources), - map_to_flatbuf(fbb, info->dynamic_resources), false); - fbb.Finish(message); - - LocalSchedulerTableSendInfoData *data = - (LocalSchedulerTableSendInfoData *) malloc( - sizeof(LocalSchedulerTableSendInfoData) + fbb.GetSize()); - data->size = fbb.GetSize(); - memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize()); - - init_table_callback(db_handle, UniqueID::nil(), __func__, - new CommonCallbackData(data), retry, NULL, - redis_local_scheduler_table_send_info, NULL); -} - -void local_scheduler_table_disconnect(DBHandle *db_handle) { - redis_local_scheduler_table_disconnect(db_handle); -} diff --git a/src/common/state/local_scheduler_table.h b/src/common/state/local_scheduler_table.h deleted file mode 100644 index 239b84d0f..000000000 --- a/src/common/state/local_scheduler_table.h +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef LOCAL_SCHEDULER_TABLE_H -#define LOCAL_SCHEDULER_TABLE_H - -#include - -#include "db.h" -#include "table.h" -#include "task.h" - -/** This struct is sent with heartbeat messages from the local scheduler to the - * global scheduler, and it contains information about the load on the local - * scheduler. */ -typedef struct { - /** The total number of workers that are connected to this local scheduler. */ - int total_num_workers; - /** The number of tasks queued in this local scheduler. */ - int task_queue_length; - /** The number of workers that are available and waiting for tasks. */ - int available_workers; - /** The resource vector of resources generally available to this local - * scheduler. */ - std::unordered_map static_resources; - /** The resource vector of resources currently available to this local - * scheduler. */ - std::unordered_map dynamic_resources; - /** Whether the local scheduler is dead. If true, then all other fields - * should be ignored. */ - bool is_dead; -} LocalSchedulerInfo; - -/* - * ==== Subscribing to the local scheduler table ==== - */ - -/* Callback for subscribing to the local scheduler table. */ -typedef void (*local_scheduler_table_subscribe_callback)( - DBClientID client_id, - LocalSchedulerInfo info, - void *user_context); - -/** - * Register a callback for a local scheduler table event. - * - * @param db_handle Database handle. - * @param subscribe_callback Callback that will be called when the local - * scheduler event happens. - * @param subscribe_context Context that will be passed into the - * subscribe_callback. - * @param retry Information about retrying the request to the database. - * @return Void. - */ -void local_scheduler_table_subscribe( - DBHandle *db_handle, - local_scheduler_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry); - -/* Data that is needed to register local scheduler table subscribe callbacks - * with the state database. */ -typedef struct { - local_scheduler_table_subscribe_callback subscribe_callback; - void *subscribe_context; -} LocalSchedulerTableSubscribeData; - -/** - * Send a heartbeat to all subscribers to the local scheduler table. This - * heartbeat contains some information about the load on the local scheduler. - * - * @param db_handle Database handle. - * @param info Information about the local scheduler, including the load on the - * local scheduler. - * @param retry Information about retrying the request to the database. - * @return Void. - */ -void local_scheduler_table_send_info(DBHandle *db_handle, - LocalSchedulerInfo *info, - RetryInfo *retry); - -/* Data that is needed to publish local scheduler heartbeats to the local - * scheduler table. */ -typedef struct { - /* The size of the flatbuffer object. */ - int64_t size; - /* The information to be sent. */ - uint8_t flatbuffer_data[0]; -} LocalSchedulerTableSendInfoData; - -/** - * Send a null heartbeat to all subscribers to the local scheduler table to - * notify them that we are about to exit. This operation is performed - * synchronously. - * - * @param db_handle Database handle. - * @return Void. - */ -void local_scheduler_table_disconnect(DBHandle *db_handle); - -#endif /* LOCAL_SCHEDULER_TABLE_H */ diff --git a/src/common/state/object_table.cc b/src/common/state/object_table.cc deleted file mode 100644 index fcd527e62..000000000 --- a/src/common/state/object_table.cc +++ /dev/null @@ -1,119 +0,0 @@ -#include "object_table.h" -#include "redis.h" - -void object_table_lookup(DBHandle *db_handle, - ObjectID object_id, - RetryInfo *retry, - object_table_lookup_done_callback done_callback, - void *user_context) { - RAY_CHECK(db_handle != NULL); - init_table_callback(db_handle, object_id, __func__, - new CommonCallbackData(NULL), retry, - (table_done_callback) done_callback, - redis_object_table_lookup, user_context); -} - -void object_table_add(DBHandle *db_handle, - ObjectID object_id, - int64_t object_size, - unsigned char digest[], - RetryInfo *retry, - object_table_done_callback done_callback, - void *user_context) { - RAY_CHECK(db_handle != NULL); - - ObjectTableAddData *info = - (ObjectTableAddData *) malloc(sizeof(ObjectTableAddData)); - info->object_size = object_size; - memcpy(&info->digest[0], digest, DIGEST_SIZE); - init_table_callback(db_handle, object_id, __func__, - new CommonCallbackData(info), retry, - (table_done_callback) done_callback, - redis_object_table_add, user_context); -} - -void object_table_remove(DBHandle *db_handle, - ObjectID object_id, - DBClientID *client_id, - RetryInfo *retry, - object_table_done_callback done_callback, - void *user_context) { - RAY_CHECK(db_handle != NULL); - /* Copy the client ID, if one was provided. */ - DBClientID *client_id_copy = NULL; - if (client_id != NULL) { - client_id_copy = (DBClientID *) malloc(sizeof(DBClientID)); - *client_id_copy = *client_id; - } - init_table_callback(db_handle, object_id, __func__, - new CommonCallbackData(client_id_copy), retry, - (table_done_callback) done_callback, - redis_object_table_remove, user_context); -} - -void object_table_subscribe_to_notifications( - DBHandle *db_handle, - bool subscribe_all, - object_table_object_available_callback object_available_callback, - void *subscribe_context, - RetryInfo *retry, - object_table_lookup_done_callback done_callback, - void *user_context) { - RAY_CHECK(db_handle != NULL); - ObjectTableSubscribeData *sub_data = - (ObjectTableSubscribeData *) malloc(sizeof(ObjectTableSubscribeData)); - sub_data->object_available_callback = object_available_callback; - sub_data->subscribe_context = subscribe_context; - sub_data->subscribe_all = subscribe_all; - - init_table_callback( - db_handle, ObjectID::nil(), __func__, new CommonCallbackData(sub_data), - retry, (table_done_callback) done_callback, - redis_object_table_subscribe_to_notifications, user_context); -} - -void object_table_request_notifications(DBHandle *db_handle, - int num_object_ids, - ObjectID object_ids[], - RetryInfo *retry) { - RAY_CHECK(db_handle != NULL); - RAY_CHECK(num_object_ids > 0); - ObjectTableRequestNotificationsData *data = - (ObjectTableRequestNotificationsData *) malloc( - sizeof(ObjectTableRequestNotificationsData) + - num_object_ids * sizeof(ObjectID)); - data->num_object_ids = num_object_ids; - memcpy(data->object_ids, object_ids, num_object_ids * sizeof(ObjectID)); - - init_table_callback(db_handle, ObjectID::nil(), __func__, - new CommonCallbackData(data), retry, NULL, - redis_object_table_request_notifications, NULL); -} - -void result_table_add(DBHandle *db_handle, - ObjectID object_id, - TaskID task_id, - bool is_put, - RetryInfo *retry, - result_table_done_callback done_callback, - void *user_context) { - ResultTableAddInfo *info = - (ResultTableAddInfo *) malloc(sizeof(ResultTableAddInfo)); - info->task_id = task_id; - info->is_put = is_put; - init_table_callback(db_handle, object_id, __func__, - new CommonCallbackData(info), retry, - (table_done_callback) done_callback, - redis_result_table_add, user_context); -} - -void result_table_lookup(DBHandle *db_handle, - ObjectID object_id, - RetryInfo *retry, - result_table_lookup_callback done_callback, - void *user_context) { - init_table_callback(db_handle, object_id, __func__, - new CommonCallbackData(NULL), retry, - (table_done_callback) done_callback, - redis_result_table_lookup, user_context); -} diff --git a/src/common/state/object_table.h b/src/common/state/object_table.h deleted file mode 100644 index 77a299dfd..000000000 --- a/src/common/state/object_table.h +++ /dev/null @@ -1,242 +0,0 @@ -#ifndef OBJECT_TABLE_H -#define OBJECT_TABLE_H - -#include "common.h" -#include "table.h" -#include "db.h" -#include "task.h" - -/* - * ==== Lookup call and callback ==== - */ - -/* Callback called when the lookup completes. The callback should free - * the manager_vector array, but NOT the strings they are pointing to. If there - * was no entry at all for the object (the object had never been created - * before), then never_created will be true. - */ -typedef void (*object_table_lookup_done_callback)( - ObjectID object_id, - bool never_created, - const std::vector &manager_ids, - void *user_context); - -/* Callback called when object ObjectID is available. */ -typedef void (*object_table_object_available_callback)( - ObjectID object_id, - int64_t data_size, - const std::vector &manager_ids, - void *user_context); - -/** - * Return the list of nodes storing object_id in their plasma stores. - * - * @param db_handle Handle to object_table database. - * @param object_id ID of the object being looked up. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Context passed by the caller. - * @return Void. - */ -void object_table_lookup(DBHandle *db_handle, - ObjectID object_id, - RetryInfo *retry, - object_table_lookup_done_callback done_callback, - void *user_context); - -/* - * ==== Add object call and callback ==== - */ - -/** - * Callback called when the object add/remove operation completes. - * - * @param object_id The ID of the object that was added or removed. - * @param success Whether the operation was successful or not. If this is false - * and the operation was an addition, the object was added, but there - * was a hash mismatch. - * @param user_context The user context that was passed into the add/remove - * call. - */ -typedef void (*object_table_done_callback)(ObjectID object_id, - bool success, - void *user_context); - -/** - * Add the plasma manager that created the db_handle to the - * list of plasma managers that have the object_id. - * - * @param db_handle Handle to db. - * @param object_id Object unique identifier. - * @param data_size Object data size. - * @param retry Information about retrying the request to the database. - * @param done_callback Callback to be called when lookup completes. - * @param user_context User context to be passed in the callbacks. - * @return Void. - */ -void object_table_add(DBHandle *db_handle, - ObjectID object_id, - int64_t object_size, - unsigned char digest[], - RetryInfo *retry, - object_table_done_callback done_callback, - void *user_context); - -/** Data that is needed to add new objects to the object table. */ -typedef struct { - int64_t object_size; - unsigned char digest[DIGEST_SIZE]; -} ObjectTableAddData; - -/* - * ==== Remove object call and callback ==== - */ - -/** - * Object remove function. - * - * @param db_handle Handle to db. - * @param object_id Object unique identifier. - * @param client_id A pointer to the database client ID to remove. If this is - * set to NULL, then the client ID associated with db_handle will be - * removed. - * @param retry Information about retrying the request to the database. - * @param done_callback Callback to be called when lookup completes. - * @param user_context User context to be passed in the callbacks. - * @return Void. - */ -void object_table_remove(DBHandle *db_handle, - ObjectID object_id, - DBClientID *client_id, - RetryInfo *retry, - object_table_done_callback done_callback, - void *user_context); - -/* - * ==== Subscribe to be announced when new object available ==== - */ - -/** - * Set up a client-specific channel for receiving notifications about available - * objects from the object table. The callback will be called once per - * notification received on this channel. - * - * @param db_handle Handle to db. - * @param object_available_callback Callback to be called when new object - * becomes available. - * @param subscribe_context Caller context which will be passed to the - * object_available_callback. - * @param retry Information about retrying the request to the database. - * @param done_callback Callback to be called when subscription is installed. - * This is only used for the tests. - * @param user_context User context to be passed into the done callback. This is - * only used for the tests. - * @return Void. - */ -void object_table_subscribe_to_notifications( - DBHandle *db_handle, - bool subscribe_all, - object_table_object_available_callback object_available_callback, - void *subscribe_context, - RetryInfo *retry, - object_table_lookup_done_callback done_callback, - void *user_context); - -/** - * Request notifications about the availability of some objects from the object - * table. The notifications will be published to this client's object - * notification channel, which was set up by the method - * object_table_subscribe_to_notifications. - * - * @param db_handle Handle to db. - * @param object_ids The object IDs to receive notifications about. - * @param retry Information about retrying the request to the database. - * @return Void. - */ -void object_table_request_notifications(DBHandle *db, - int num_object_ids, - ObjectID object_ids[], - RetryInfo *retry); - -/** Data that is needed to run object_request_notifications requests. */ -typedef struct { - /** The number of object IDs. */ - int num_object_ids; - /** This field is used to store a variable number of object IDs. */ - ObjectID object_ids[0]; -} ObjectTableRequestNotificationsData; - -/** Data that is needed to register new object available callbacks with the - * state database. */ -typedef struct { - bool subscribe_all; - object_table_object_available_callback object_available_callback; - void *subscribe_context; -} ObjectTableSubscribeData; - -/* - * ==== Result table ==== - */ - -/** - * Callback called when the add/remove operation for a result table entry - * completes. */ -typedef void (*result_table_done_callback)(ObjectID object_id, - void *user_context); - -/** Information about a result table entry to add. */ -typedef struct { - /** The task ID of the task that created the requested object. */ - TaskID task_id; - /** True if the object was created through a put, and false if created by - * return value. */ - bool is_put; -} ResultTableAddInfo; - -/** - * Add information about a new object to the object table. This - * is immutable information like the ID of the task that - * created the object. - * - * @param db_handle Handle to object_table database. - * @param object_id ID of the object to add. - * @param task_id ID of the task that creates this object. - * @param is_put A boolean that is true if the object was created through a - * ray.put, and false if the object was created by return value. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Context passed by the caller. - * @return Void. - */ -void result_table_add(DBHandle *db_handle, - ObjectID object_id, - TaskID task_id, - bool is_put, - RetryInfo *retry, - result_table_done_callback done_callback, - void *user_context); - -/** Callback called when the result table lookup completes. */ -typedef void (*result_table_lookup_callback)(ObjectID object_id, - TaskID task_id, - bool is_put, - void *user_context); - -/** - * Lookup the task that created an object in the result table. The return value - * is the task ID. - * - * @param db_handle Handle to object_table database. - * @param object_id ID of the object to lookup. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Context passed by the caller. - * @return Void. - */ -void result_table_lookup(DBHandle *db_handle, - ObjectID object_id, - RetryInfo *retry, - result_table_lookup_callback done_callback, - void *user_context); - -#endif /* OBJECT_TABLE_H */ diff --git a/src/common/state/redis.cc b/src/common/state/redis.cc deleted file mode 100644 index 17a3c8ce2..000000000 --- a/src/common/state/redis.cc +++ /dev/null @@ -1,1692 +0,0 @@ -/* Redis implementation of the global state store */ - -#include -#include -#include -#include - -extern "C" { -/* Including hiredis here is necessary on Windows for typedefs used in ae.h. */ -#include "hiredis/hiredis.h" -#include "hiredis/adapters/ae.h" -} - -#include "common.h" -#include "db.h" -#include "db_client_table.h" -#include "actor_notification_table.h" -#include "driver_table.h" -#include "local_scheduler_table.h" -#include "object_table.h" -#include "task.h" -#include "task_table.h" -#include "error_table.h" -#include "event_loop.h" -#include "redis.h" -#include "io.h" -#include "net.h" - -#include "format/common_generated.h" - -#include "common_protocol.h" - -#ifndef _WIN32 -/* This function is actually not declared in standard POSIX, so declare it. */ -extern int usleep(useconds_t usec); -#endif - -#define CHECK_REDIS_CONNECT(CONTEXT_TYPE, context, M, ...) \ - do { \ - CONTEXT_TYPE *_context = (context); \ - if (!_context) { \ - RAY_LOG(FATAL) << "could not allocate redis context"; \ - } \ - if (_context->err) { \ - RAY_LOG(ERROR) << M; \ - LOG_REDIS_ERROR(_context, ""); \ - exit(-1); \ - } \ - } while (0) - -/** - * A header for callbacks of a single Redis asynchronous command. The user must - * pass in the table operation's timer ID as the asynchronous command's - * privdata field when executing the asynchronous command. The user must define - * variable names for DB and CB_DATA. After this piece of code runs, DB - * will hold a reference to the database handle, CB_DATA will hold a reference - * to the callback data for this table operation. The user must pass in the - * redisReply pointer as the REPLY argument. - * - * This header also short-circuits the entire callback if: (1) there was no - * reply from Redis, or (2) the callback data for this table operation was - * already removed, meaning that the operation was already marked as succeeded - * or failed. - */ -#define REDIS_CALLBACK_HEADER(DB, CB_DATA, REPLY) \ - if ((REPLY) == NULL) { \ - return; \ - } \ - DBHandle *DB = (DBHandle *) c->data; \ - TableCallbackData *CB_DATA = outstanding_callbacks_find((int64_t) privdata); \ - if (CB_DATA == NULL) { \ - /* the callback data structure has been \ - * already freed; just ignore this reply */ \ - return; \ - } \ - do { \ - } while (0) - -redisAsyncContext *get_redis_context(DBHandle *db, UniqueID id) { - /* NOTE: The hash function used here must match the one in - * PyObjectID_redis_shard_hash in src/common/lib/python/common_extension.cc. - * Changes to the hash function should only be made through - * std::hash in src/common/common.h */ - std::hash index; - return db->contexts[index(id) % db->contexts.size()]; -} - -redisAsyncContext *get_redis_subscribe_context(DBHandle *db, UniqueID id) { - std::hash index; - return db->subscribe_contexts[index(id) % db->subscribe_contexts.size()]; -} - -void get_redis_shards(redisContext *context, - std::vector &db_shards_addresses, - std::vector &db_shards_ports) { - /* Get the total number of Redis shards in the system. */ - int num_attempts = 0; - redisReply *reply = NULL; - while (num_attempts < RayConfig::instance().redis_db_connect_retries()) { - /* Try to read the number of Redis shards from the primary shard. If the - * entry is present, exit. */ - reply = (redisReply *) redisCommand(context, "GET NumRedisShards"); - if (reply->type != REDIS_REPLY_NIL) { - break; - } - - /* Sleep for a little, and try again if the entry isn't there yet. */ - freeReplyObject(reply); - usleep(RayConfig::instance().redis_db_connect_wait_milliseconds() * 1000); - num_attempts++; - continue; - } - RAY_CHECK(num_attempts < RayConfig::instance().redis_db_connect_retries()) - << "No entry found for NumRedisShards"; - RAY_CHECK(reply->type == REDIS_REPLY_STRING) - << "Expected string, found Redis type " << reply->type - << " for NumRedisShards"; - int num_redis_shards = atoi(reply->str); - RAY_CHECK(num_redis_shards >= 1) << "Expected at least one Redis shard, " - << "found " << num_redis_shards; - freeReplyObject(reply); - - /* Get the addresses of all of the Redis shards. */ - num_attempts = 0; - while (num_attempts < RayConfig::instance().redis_db_connect_retries()) { - /* Try to read the Redis shard locations from the primary shard. If we find - * that all of them are present, exit. */ - reply = (redisReply *) redisCommand(context, "LRANGE RedisShards 0 -1"); - if (static_cast(reply->elements) == num_redis_shards) { - break; - } - - /* Sleep for a little, and try again if not all Redis shard addresses have - * been added yet. */ - freeReplyObject(reply); - usleep(RayConfig::instance().redis_db_connect_wait_milliseconds() * 1000); - num_attempts++; - continue; - } - RAY_CHECK(num_attempts < RayConfig::instance().redis_db_connect_retries()) - << "Expected " << num_redis_shards << " Redis shard addresses, found " - << reply->elements; - - /* Parse the Redis shard addresses. */ - char db_shard_address[16]; - int db_shard_port; - for (size_t i = 0; i < reply->elements; ++i) { - /* Parse the shard addresses and ports. */ - RAY_CHECK(reply->element[i]->type == REDIS_REPLY_STRING); - RAY_CHECK(parse_ip_addr_port(reply->element[i]->str, db_shard_address, - &db_shard_port) == 0); - db_shards_addresses.push_back(std::string(db_shard_address)); - db_shards_ports.push_back(db_shard_port); - } - freeReplyObject(reply); -} - -void db_connect_shard(const std::string &db_address, - int db_port, - DBClientID client, - const char *client_type, - const char *node_ip_address, - const std::vector &args, - DBHandle *db, - redisAsyncContext **context_out, - redisAsyncContext **subscribe_context_out, - redisContext **sync_context_out) { - /* Synchronous connection for initial handshake */ - redisReply *reply; - int connection_attempts = 0; - redisContext *sync_context = redisConnect(db_address.c_str(), db_port); - while (sync_context == NULL || sync_context->err) { - if (connection_attempts >= - RayConfig::instance().redis_db_connect_retries()) { - break; - } - RAY_LOG(WARNING) << "Failed to connect to Redis, retrying."; - /* Sleep for a little. */ - usleep(RayConfig::instance().redis_db_connect_wait_milliseconds() * 1000); - sync_context = redisConnect(db_address.c_str(), db_port); - connection_attempts += 1; - } - CHECK_REDIS_CONNECT(redisContext, sync_context, - "could not establish synchronous connection to redis " - "%s:%d", - db_address.c_str(), db_port); - /* Configure Redis to generate keyspace notifications for list events. This - * should only need to be done once (by whoever started Redis), but since - * Redis may be started in multiple places (e.g., for testing or when starting - * processes by hand), it is easier to do it multiple times. */ - reply = (redisReply *) redisCommand(sync_context, - "CONFIG SET notify-keyspace-events Kl"); - RAY_CHECK(reply != NULL) << "db_connect failed on CONFIG SET"; - freeReplyObject(reply); - /* Also configure Redis to not run in protected mode, so clients on other - * hosts can connect to it. */ - reply = - (redisReply *) redisCommand(sync_context, "CONFIG SET protected-mode no"); - RAY_CHECK(reply != NULL) << "db_connect failed on CONFIG SET"; - freeReplyObject(reply); - - /* Construct the argument arrays for RAY.CONNECT. */ - int argc = args.size() + 4; - const char **argv = (const char **) malloc(sizeof(char *) * argc); - size_t *argvlen = (size_t *) malloc(sizeof(size_t) * argc); - /* Set the command name argument. */ - argv[0] = "RAY.CONNECT"; - argvlen[0] = strlen(argv[0]); - /* Set the client ID argument. */ - argv[1] = (char *) client.data(); - argvlen[1] = sizeof(client); - /* Set the node IP address argument. */ - argv[2] = node_ip_address; - argvlen[2] = strlen(node_ip_address); - /* Set the client type argument. */ - argv[3] = client_type; - argvlen[3] = strlen(client_type); - /* Set the remaining arguments. */ - for (size_t i = 0; i < args.size(); ++i) { - argv[4 + i] = args[i].c_str(); - argvlen[4 + i] = strlen(args[i].c_str()); - } - - /* Register this client with Redis. RAY.CONNECT is a custom Redis command that - * we've defined. */ - reply = (redisReply *) redisCommandArgv(sync_context, argc, argv, argvlen); - RAY_CHECK(reply != NULL) << "db_connect failed on RAY.CONNECT"; - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - freeReplyObject(reply); - free(argv); - free(argvlen); - - *sync_context_out = sync_context; - - /* Establish connection for control data. */ - redisAsyncContext *context = redisAsyncConnect(db_address.c_str(), db_port); - CHECK_REDIS_CONNECT(redisAsyncContext, context, - "could not establish asynchronous connection to redis " - "%s:%d", - db_address.c_str(), db_port); - context->data = (void *) db; - *context_out = context; - - /* Establish async connection for subscription. */ - redisAsyncContext *subscribe_context = - redisAsyncConnect(db_address.c_str(), db_port); - CHECK_REDIS_CONNECT(redisAsyncContext, subscribe_context, - "could not establish asynchronous subscription " - "connection to redis %s:%d", - db_address.c_str(), db_port); - subscribe_context->data = (void *) db; - *subscribe_context_out = subscribe_context; -} - -DBHandle *db_connect(const std::string &db_primary_address, - int db_primary_port, - const char *client_type, - const char *node_ip_address, - const std::vector &args) { - /* Check that the number of args is even. These args will be passed to the - * RAY.CONNECT Redis command, which takes arguments in pairs. */ - if (args.size() % 2 != 0) { - RAY_LOG(FATAL) << "The number of extra args must be divisible by two."; - } - - /* Create a client ID for this client. */ - DBClientID client = DBClientID::from_random(); - - DBHandle *db = new DBHandle(); - - db->client_type = strdup(client_type); - db->client = client; - - redisAsyncContext *context; - redisAsyncContext *subscribe_context; - redisContext *sync_context; - - /* Connect to the primary redis instance. */ - db_connect_shard(db_primary_address, db_primary_port, client, client_type, - node_ip_address, args, db, &context, &subscribe_context, - &sync_context); - db->context = context; - db->subscribe_context = subscribe_context; - db->sync_context = sync_context; - - /* Get the shard locations. */ - std::vector db_shards_addresses; - std::vector db_shards_ports; - get_redis_shards(db->sync_context, db_shards_addresses, db_shards_ports); - RAY_CHECK(db_shards_addresses.size() > 0) << "No Redis shards found"; - /* Connect to the shards. */ - for (size_t i = 0; i < db_shards_addresses.size(); ++i) { - db_connect_shard(db_shards_addresses[i], db_shards_ports[i], client, - client_type, node_ip_address, args, db, &context, - &subscribe_context, &sync_context); - db->contexts.push_back(context); - db->subscribe_contexts.push_back(subscribe_context); - redisFree(sync_context); - } - - return db; -} - -void DBHandle_free(DBHandle *db) { - /* Clean up the primary Redis connection state. */ - redisFree(db->sync_context); - redisAsyncFree(db->context); - redisAsyncFree(db->subscribe_context); - - /* Clean up the Redis shards. */ - RAY_CHECK(db->contexts.size() == db->subscribe_contexts.size()); - for (size_t i = 0; i < db->contexts.size(); ++i) { - redisAsyncFree(db->contexts[i]); - redisAsyncFree(db->subscribe_contexts[i]); - } - - free(db->client_type); - delete db; -} - -void db_disconnect(DBHandle *db) { - /* Notify others that this client is disconnecting from Redis. If a client of - * the same type on the same node wants to reconnect again, they must - * reconnect and get assigned a different client ID. */ - redisReply *reply = - (redisReply *) redisCommand(db->sync_context, "RAY.DISCONNECT %b", - db->client.data(), sizeof(db->client)); - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - freeReplyObject(reply); - - DBHandle_free(db); -} - -void db_attach(DBHandle *db, event_loop *loop, bool reattach) { - db->loop = loop; - /* Attach primary redis instance to the event loop. */ - int err = redisAeAttach(loop, db->context); - /* If the database is reattached in the tests, redis normally gives - * an error which we can safely ignore. */ - if (!reattach) { - RAY_CHECK(err == REDIS_OK) << "failed to attach the event loop"; - } - err = redisAeAttach(loop, db->subscribe_context); - if (!reattach) { - RAY_CHECK(err == REDIS_OK) << "failed to attach the event loop"; - } - /* Attach other redis shards to the event loop. */ - RAY_CHECK(db->contexts.size() == db->subscribe_contexts.size()); - for (size_t i = 0; i < db->contexts.size(); ++i) { - int err = redisAeAttach(loop, db->contexts[i]); - /* If the database is reattached in the tests, redis normally gives - * an error which we can safely ignore. */ - if (!reattach) { - RAY_CHECK(err == REDIS_OK) << "failed to attach the event loop"; - } - err = redisAeAttach(loop, db->subscribe_contexts[i]); - if (!reattach) { - RAY_CHECK(err == REDIS_OK) << "failed to attach the event loop"; - } - } -} - -/* - * ==== object_table callbacks ==== - */ - -void redis_object_table_add_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - /* Do some minimal checking. */ - redisReply *reply = (redisReply *) r; - bool success = (strcmp(reply->str, "hash mismatch") != 0); - if (!success) { - /* If our object hash doesn't match the one recorded in the table, report - * the error back to the user and exit immediately. */ - RAY_LOG(WARNING) << "Found objects with different value but same object " - << "ID, most likely because a nondeterministic task was " - << "executed twice, either for reconstruction or for " - << "speculation."; - } else { - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " - << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - } - /* Call the done callback if there is one. */ - if (callback_data->done_callback != NULL) { - object_table_done_callback done_callback = - (object_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, success, callback_data->user_context); - } - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_object_table_add(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - - ObjectTableAddData *info = (ObjectTableAddData *) callback_data->data->Get(); - ObjectID obj_id = callback_data->id; - int64_t object_size = info->object_size; - unsigned char *digest = info->digest; - - redisAsyncContext *context = get_redis_context(db, obj_id); - - int status = redisAsyncCommand( - context, redis_object_table_add_callback, - (void *) callback_data->timer_id, "RAY.OBJECT_TABLE_ADD %b %lld %b %b", - obj_id.data(), sizeof(obj_id), (long long) object_size, digest, - (size_t) DIGEST_SIZE, db->client.data(), sizeof(db->client)); - - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_object_table_add"); - } -} - -void redis_object_table_remove_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - /* Do some minimal checking. */ - redisReply *reply = (redisReply *) r; - if (strcmp(reply->str, "object not found") == 0) { - /* If our object entry was not in the table, it's probably a race - * condition with an object_table_add. */ - return; - } - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - /* Call the done callback if there is one. */ - if (callback_data->done_callback != NULL) { - object_table_done_callback done_callback = - (object_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, true, callback_data->user_context); - } - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_object_table_remove(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - - ObjectID obj_id = callback_data->id; - /* If the caller provided a manager ID to delete, use it. Otherwise, use our - * own client ID as the ID to delete. */ - DBClientID *client_id = (DBClientID *) callback_data->data->Get(); - if (client_id == NULL) { - client_id = &db->client; - } - - redisAsyncContext *context = get_redis_context(db, obj_id); - - int status = redisAsyncCommand( - context, redis_object_table_remove_callback, - (void *) callback_data->timer_id, "RAY.OBJECT_TABLE_REMOVE %b %b", - obj_id.data(), sizeof(obj_id), client_id->data(), sizeof(*client_id)); - - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_object_table_remove"); - } -} - -void redis_object_table_lookup(TableCallbackData *callback_data) { - RAY_CHECK(callback_data); - DBHandle *db = callback_data->db_handle; - - ObjectID obj_id = callback_data->id; - - redisAsyncContext *context = get_redis_context(db, obj_id); - - int status = redisAsyncCommand(context, redis_object_table_lookup_callback, - (void *) callback_data->timer_id, - "RAY.OBJECT_TABLE_LOOKUP %b", obj_id.data(), - sizeof(obj_id)); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in object_table lookup"); - } -} - -void redis_result_table_add_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - /* Check that the command succeeded. */ - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strncmp(reply->str, "OK", strlen("OK")) == 0) << "reply->str is " - << reply->str; - /* Call the done callback if there is one. */ - if (callback_data->done_callback) { - result_table_done_callback done_callback = - (result_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, callback_data->user_context); - } - destroy_timer_callback(db->loop, callback_data); -} - -void redis_result_table_add(TableCallbackData *callback_data) { - RAY_CHECK(callback_data); - DBHandle *db = callback_data->db_handle; - ObjectID id = callback_data->id; - ResultTableAddInfo *info = (ResultTableAddInfo *) callback_data->data->Get(); - int is_put = info->is_put ? 1 : 0; - - redisAsyncContext *context = get_redis_context(db, id); - - /* Add the result entry to the result table. */ - int status = - redisAsyncCommand(context, redis_result_table_add_callback, - (void *) callback_data->timer_id, - "RAY.RESULT_TABLE_ADD %b %b %d", id.data(), sizeof(id), - info->task_id.data(), sizeof(info->task_id), is_put); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "Error in result table add"); - } -} - -/* This allocates a task which must be freed by the caller, unless the returned - * task is NULL. This is used by both redis_result_table_lookup_callback and - * redis_task_table_get_task_callback. */ -Task *parse_and_construct_task_from_redis_reply(redisReply *reply) { - Task *task = NULL; - if (reply->type == REDIS_REPLY_NIL) { - /* There is no task in the reply, so return NULL. */ - } else if (reply->type == REDIS_REPLY_STRING) { - /* The reply is a flatbuffer TaskReply object. Parse it and construct the - * task. */ - auto message = flatbuffers::GetRoot(reply->str); - TaskSpec *spec = (TaskSpec *) message->task_spec()->data(); - int64_t task_spec_size = message->task_spec()->size(); - auto execution_dependencies = - flatbuffers::GetRoot( - message->execution_dependencies()->data()); - task = Task_alloc( - spec, task_spec_size, static_cast(message->state()), - from_flatbuf(*message->local_scheduler_id()), - from_flatbuf(*execution_dependencies->execution_dependencies())); - } else { - RAY_LOG(FATAL) << "Unexpected reply type " << reply->type; - } - /* Return the task. If it is not NULL, then it must be freed by the caller. */ - return task; -} - -void redis_result_table_lookup_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_STRING) - << "Unexpected reply type " << reply->type << " in " - << "redis_result_table_lookup_callback"; - /* Parse the task from the reply. */ - TaskID result_id = TaskID::nil(); - bool is_put = false; - if (reply->type == REDIS_REPLY_STRING) { - auto message = flatbuffers::GetRoot(reply->str); - result_id = from_flatbuf(*message->task_id()); - is_put = message->is_put(); - } - - /* Call the done callback if there is one. */ - result_table_lookup_callback done_callback = - (result_table_lookup_callback) callback_data->done_callback; - if (done_callback != NULL) { - done_callback(callback_data->id, result_id, is_put, - callback_data->user_context); - } - /* Clean up timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_result_table_lookup(TableCallbackData *callback_data) { - RAY_CHECK(callback_data); - DBHandle *db = callback_data->db_handle; - ObjectID id = callback_data->id; - redisAsyncContext *context = get_redis_context(db, id); - int status = - redisAsyncCommand(context, redis_result_table_lookup_callback, - (void *) callback_data->timer_id, - "RAY.RESULT_TABLE_LOOKUP %b", id.data(), sizeof(id)); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "Error in result table lookup"); - } -} - -DBClient redis_db_client_table_get(DBHandle *db, - const unsigned char *client_id, - size_t client_id_len) { - redisReply *reply = - (redisReply *) redisCommand(db->sync_context, "HGETALL %s%b", - DB_CLIENT_PREFIX, client_id, client_id_len); - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements > 0); - DBClient db_client; - int num_fields = 0; - /* Parse the fields into a DBClient. */ - for (size_t j = 0; j < reply->elements; j = j + 2) { - const char *key = reply->element[j]->str; - const char *value = reply->element[j + 1]->str; - if (strcmp(key, "ray_client_id") == 0) { - memcpy(db_client.id.mutable_data(), value, sizeof(db_client.id)); - num_fields++; - } else if (strcmp(key, "client_type") == 0) { - db_client.client_type = std::string(value); - num_fields++; - } else if (strcmp(key, "manager_address") == 0) { - db_client.manager_address = std::string(value); - num_fields++; - } else if (strcmp(key, "deleted") == 0) { - bool is_deleted = atoi(value); - db_client.is_alive = !is_deleted; - num_fields++; - } - } - freeReplyObject(reply); - /* The client ID, type, and whether it is deleted are all - * mandatory fields. Auxiliary address is optional. */ - RAY_CHECK(num_fields >= 3); - return db_client; -} - -void redis_cache_set_db_client(DBHandle *db, DBClient client) { - db->db_client_cache[client.id] = client; -} - -/** - * Get an entry from the plasma manager table in redis. - * - * @param db The database handle. - * @param index The index of the plasma manager. - * @return The IP address and port of the manager. - */ -DBClient redis_cache_get_db_client(DBHandle *db, DBClientID db_client_id) { - auto it = db->db_client_cache.find(db_client_id); - if (it == db->db_client_cache.end()) { - DBClient db_client = redis_db_client_table_get(db, db_client_id.data(), - sizeof(db_client_id)); - db->db_client_cache[db_client_id] = db_client; - it = db->db_client_cache.find(db_client_id); - } - return it->second; -} - -void redis_object_table_lookup_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - RAY_LOG(DEBUG) << "Object table lookup callback"; - RAY_CHECK(reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ARRAY); - - object_table_lookup_done_callback done_callback = - (object_table_lookup_done_callback) callback_data->done_callback; - - ObjectID obj_id = callback_data->id; - - /* Parse the Redis reply. */ - if (reply->type == REDIS_REPLY_NIL) { - /* The object entry did not exist. */ - if (done_callback) { - done_callback(obj_id, true, std::vector(), - callback_data->user_context); - } - } else if (reply->type == REDIS_REPLY_ARRAY) { - /* Extract the manager IDs from the response into a vector. */ - std::vector manager_ids; - - for (size_t j = 0; j < reply->elements; ++j) { - RAY_CHECK(reply->element[j]->type == REDIS_REPLY_STRING); - DBClientID manager_id; - memcpy(manager_id.mutable_data(), reply->element[j]->str, - sizeof(manager_id)); - manager_ids.push_back(manager_id); - } - - if (done_callback) { - done_callback(obj_id, false, manager_ids, callback_data->user_context); - } - } else { - RAY_LOG(FATAL) << "Unexpected reply type from object table lookup."; - } - - /* Clean up timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void object_table_redis_subscribe_to_notifications_callback( - redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - /* Replies to the SUBSCRIBE command have 3 elements. There are two - * possibilities. Either the reply is the initial acknowledgment of the - * subscribe command, or it is a message. If it is the initial acknowledgment, - * then - * - reply->element[0]->str is "subscribe" - * - reply->element[1]->str is the name of the channel - * - reply->emement[2]->str is null. - * If it is an actual message, then - * - reply->element[0]->str is "message" - * - reply->element[1]->str is the name of the channel - * - reply->emement[2]->str is the contents of the message. - */ - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements == 3); - redisReply *message_type = reply->element[0]; - RAY_LOG(DEBUG) << "Object table subscribe to notifications callback, message" - << message_type->str; - - if (strcmp(message_type->str, "message") == 0) { - /* We received an object notification. Parse the payload. */ - auto message = flatbuffers::GetRoot( - reply->element[2]->str); - /* Extract the object ID. */ - ObjectID obj_id = from_flatbuf(*message->object_id()); - /* Extract the data size. */ - int64_t data_size = message->object_size(); - int manager_count = message->manager_ids()->size(); - - /* Extract the manager IDs from the response into a vector. */ - std::vector manager_ids; - for (int i = 0; i < manager_count; ++i) { - DBClientID manager_id = from_flatbuf(*message->manager_ids()->Get(i)); - manager_ids.push_back(manager_id); - } - - /* Call the subscribe callback. */ - ObjectTableSubscribeData *data = - (ObjectTableSubscribeData *) callback_data->data->Get(); - if (data->object_available_callback) { - data->object_available_callback(obj_id, data_size, manager_ids, - data->subscribe_context); - } - } else if (strcmp(message_type->str, "subscribe") == 0) { - /* The reply for the initial SUBSCRIBE command. */ - /* Call the done callback if there is one. This code path should only be - * used in the tests. */ - if (callback_data->done_callback != NULL) { - object_table_lookup_done_callback done_callback = - (object_table_lookup_done_callback) callback_data->done_callback; - done_callback(ray::UniqueID::nil(), false, std::vector(), - callback_data->user_context); - } - /* If the initial SUBSCRIBE was successful, clean up the timer, but don't - * destroy the callback data. */ - remove_timer_callback(db->loop, callback_data); - } else { - RAY_LOG(FATAL) << "Unexpected reply type from object table subscribe to " - << "notifications."; - } -} - -void redis_object_table_subscribe_to_notifications( - TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - /* The object channel prefix must match the value defined in - * src/common/redismodule/ray_redis_module.cc. */ - const char *object_channel_prefix = "OC:"; - const char *object_channel_bcast = "BCAST"; - for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) { - int status = REDIS_OK; - /* Subscribe to notifications from the object table. This uses the client ID - * as the channel name so this channel is specific to this client. - * TODO(rkn): - * The channel name should probably be the client ID with some prefix. */ - RAY_CHECK(callback_data->data->Get() != NULL) - << "Object table subscribe data passed as NULL."; - if (((ObjectTableSubscribeData *) (callback_data->data->Get())) - ->subscribe_all) { - /* Subscribe to the object broadcast channel. */ - status = redisAsyncCommand( - db->subscribe_contexts[i], - object_table_redis_subscribe_to_notifications_callback, - (void *) callback_data->timer_id, "SUBSCRIBE %s%s", - object_channel_prefix, object_channel_bcast); - } else { - status = redisAsyncCommand( - db->subscribe_contexts[i], - object_table_redis_subscribe_to_notifications_callback, - (void *) callback_data->timer_id, "SUBSCRIBE %s%b", - object_channel_prefix, db->client.data(), sizeof(db->client)); - } - - if ((status == REDIS_ERR) || db->subscribe_contexts[i]->err) { - LOG_REDIS_DEBUG(db->subscribe_contexts[i], - "error in redis_object_table_subscribe_to_notifications"); - } - } -} - -void redis_object_table_request_notifications_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - /* Do some minimal checking. */ - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - RAY_CHECK(callback_data->done_callback == NULL); - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_object_table_request_notifications( - TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - - ObjectTableRequestNotificationsData *request_data = - (ObjectTableRequestNotificationsData *) callback_data->data->Get(); - int num_object_ids = request_data->num_object_ids; - ObjectID *object_ids = request_data->object_ids; - - for (int i = 0; i < num_object_ids; ++i) { - redisAsyncContext *context = get_redis_context(db, object_ids[i]); - - /* Create the arguments for the Redis command. */ - int num_args = 1 + 1 + 1; - const char **argv = (const char **) malloc(sizeof(char *) * num_args); - size_t *argvlen = (size_t *) malloc(sizeof(size_t) * num_args); - /* Set the command name argument. */ - argv[0] = "RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS"; - argvlen[0] = strlen(argv[0]); - /* Set the client ID argument. */ - argv[1] = (char *) db->client.data(); - argvlen[1] = sizeof(db->client); - /* Set the object ID arguments. */ - argv[2] = (char *) object_ids[i].data(); - argvlen[2] = sizeof(object_ids[i]); - - int status = redisAsyncCommandArgv( - context, redis_object_table_request_notifications_callback, - (void *) callback_data->timer_id, num_args, argv, argvlen); - free(argv); - free(argvlen); - - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, - "error in redis_object_table_subscribe_to_notifications"); - } - } -} - -/* - * ==== task_table callbacks ==== - */ - -void redis_task_table_get_task_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - /* Parse the task from the reply. */ - Task *task = parse_and_construct_task_from_redis_reply(reply); - /* Call the done callback if there is one. */ - task_table_get_callback done_callback = - (task_table_get_callback) callback_data->done_callback; - if (done_callback != NULL) { - done_callback(task, callback_data->user_context); - } - /* Free the task if it is not NULL. */ - if (task != NULL) { - Task_free(task); - } - - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_task_table_get_task(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - RAY_CHECK(callback_data->data->Get() == NULL); - TaskID task_id = callback_data->id; - - redisAsyncContext *context = get_redis_context(db, task_id); - - int status = redisAsyncCommand(context, redis_task_table_get_task_callback, - (void *) callback_data->timer_id, - "RAY.TASK_TABLE_GET %b", task_id.data(), - sizeof(task_id)); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_task_table_get_task"); - } -} - -void redis_task_table_add_task_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - // If no subscribers received the message, call the failure callback. The - // caller should decide whether to retry the add. NOTE(swang): The caller - // should check whether the receiving subscriber is still alive in the - // db_client table before retrying the add. - if (reply->type == REDIS_REPLY_ERROR && - strcmp(reply->str, "No subscribers received message.") == 0) { - RAY_LOG(WARNING) << "No subscribers received the task_table_add message."; - if (callback_data->retry.fail_callback != NULL) { - callback_data->retry.fail_callback(callback_data->id, - callback_data->user_context, - callback_data->data->Get()); - } - } else { - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " - << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - /* Call the done callback if there is one. */ - if (callback_data->done_callback != NULL) { - task_table_done_callback done_callback = - (task_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, callback_data->user_context); - } - } - - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_task_table_add_task(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - Task *task = (Task *) callback_data->data->Get(); - RAY_CHECK(task != NULL) << "NULL task passed to redis_task_table_add_task."; - - TaskID task_id = Task_task_id(task); - DBClientID local_scheduler_id = Task_local_scheduler(task); - redisAsyncContext *context = get_redis_context(db, task_id); - int state = static_cast(Task_state(task)); - - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - TaskSpec *spec = execution_spec->Spec(); - - flatbuffers::FlatBufferBuilder fbb; - auto execution_dependencies = CreateTaskExecutionDependencies( - fbb, to_flatbuf(fbb, execution_spec->ExecutionDependencies())); - fbb.Finish(execution_dependencies); - - int status = redisAsyncCommand( - context, redis_task_table_add_task_callback, - (void *) callback_data->timer_id, "RAY.TASK_TABLE_ADD %b %d %b %b %d %b", - task_id.data(), sizeof(task_id), state, local_scheduler_id.data(), - sizeof(local_scheduler_id), fbb.GetBufferPointer(), - (size_t) fbb.GetSize(), - static_cast(execution_spec->SpillbackCount()), spec, - execution_spec->SpecSize()); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_task_table_add_task"); - } -} - -void redis_task_table_update_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - // If no subscribers received the message, call the failure callback. The - // caller should decide whether to retry the update. NOTE(swang): Retrying a - // task table update can race with the liveness monitor. Do not retry the - // update unless the caller is sure that the receiving subscriber is still - // alive in the db_client table. - if (reply->type == REDIS_REPLY_ERROR) { - RAY_LOG(WARNING) << "task_table_update failed with " << reply->str; - if (callback_data->retry.fail_callback != NULL) { - callback_data->retry.fail_callback(callback_data->id, - callback_data->user_context, - callback_data->data->Get()); - } else { - RAY_LOG(FATAL) << "task_table_update failed and no fail_callback is set"; - } - } else { - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - - /* Call the done callback if there is one. */ - if (callback_data->done_callback != NULL) { - task_table_done_callback done_callback = - (task_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, callback_data->user_context); - } - } - - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_task_table_update(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - Task *task = (Task *) callback_data->data->Get(); - RAY_CHECK(task != NULL) << "NULL task passed to redis_task_table_update."; - - TaskID task_id = Task_task_id(task); - redisAsyncContext *context = get_redis_context(db, task_id); - DBClientID local_scheduler_id = Task_local_scheduler(task); - int state = static_cast(Task_state(task)); - - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - flatbuffers::FlatBufferBuilder fbb; - auto execution_dependencies = CreateTaskExecutionDependencies( - fbb, to_flatbuf(fbb, execution_spec->ExecutionDependencies())); - fbb.Finish(execution_dependencies); - - int status = redisAsyncCommand( - context, redis_task_table_update_callback, - (void *) callback_data->timer_id, "RAY.TASK_TABLE_UPDATE %b %d %b %b %d", - task_id.data(), sizeof(task_id), state, local_scheduler_id.data(), - sizeof(local_scheduler_id), fbb.GetBufferPointer(), - (size_t) fbb.GetSize(), - static_cast(execution_spec->SpillbackCount())); - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_task_table_update"); - } -} - -void redis_task_table_test_and_update_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - /* Parse the task from the reply. */ - Task *task = parse_and_construct_task_from_redis_reply(reply); - if (task == NULL) { - /* A NULL task means that the task was not in the task table. NOTE(swang): - * For normal tasks, this is not expected behavior, but actor tasks may be - * delayed when added to the task table if they are submitted to a local - * scheduler before it receives the notification that maps the actor to a - * local scheduler. */ - RAY_LOG(ERROR) << "No task found during task_table_test_and_update for " - << "task with ID " << callback_data->id; - return; - } - /* Determine whether the update happened. */ - auto message = flatbuffers::GetRoot(reply->str); - bool updated = message->updated(); - - /* Call the done callback if there is one. */ - task_table_test_and_update_callback done_callback = - (task_table_test_and_update_callback) callback_data->done_callback; - if (done_callback != NULL) { - done_callback(task, callback_data->user_context, updated); - } - /* Free the task if it is not NULL. */ - if (task != NULL) { - Task_free(task); - } - /* Clean up timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_task_table_test_and_update(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - TaskID task_id = callback_data->id; - redisAsyncContext *context = get_redis_context(db, task_id); - TaskTableTestAndUpdateData *update_data = - (TaskTableTestAndUpdateData *) callback_data->data->Get(); - - int status; - /* If the test local scheduler ID is NIL, then ignore it. */ - if (update_data->test_local_scheduler_id.is_nil()) { - status = redisAsyncCommand( - context, redis_task_table_test_and_update_callback, - (void *) callback_data->timer_id, - "RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b", task_id.data(), - sizeof(task_id), update_data->test_state_bitmask, - update_data->update_state, update_data->local_scheduler_id.data(), - sizeof(update_data->local_scheduler_id)); - } else { - status = redisAsyncCommand( - context, redis_task_table_test_and_update_callback, - (void *) callback_data->timer_id, - "RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b %b", task_id.data(), - sizeof(task_id), update_data->test_state_bitmask, - update_data->update_state, update_data->local_scheduler_id.data(), - sizeof(update_data->local_scheduler_id), - update_data->test_local_scheduler_id.data(), - sizeof(update_data->test_local_scheduler_id)); - } - - if ((status == REDIS_ERR) || context->err) { - LOG_REDIS_DEBUG(context, "error in redis_task_table_test_and_update"); - } -} - -void redis_task_table_subscribe_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - /* The number of elements is 3 for a reply to SUBSCRIBE, and 4 for a reply to - * PSUBSCRIBE. */ - RAY_CHECK(reply->elements == 3 || reply->elements == 4) - << "reply->elements is " << reply->elements; - /* The first element is the message type and the last entry is the payload. - * The middle one or middle two elements describe the channel that was - * published on. */ - redisReply *message_type = reply->element[0]; - redisReply *payload = reply->element[reply->elements - 1]; - if (strcmp(message_type->str, "message") == 0 || - strcmp(message_type->str, "pmessage") == 0) { - /* Handle a task table event. Parse the payload and call the callback. */ - auto message = flatbuffers::GetRoot(payload->str); - /* Extract the scheduling state. */ - TaskStatus state = static_cast(message->state()); - /* Extract the local scheduler ID. */ - DBClientID local_scheduler_id = - from_flatbuf(*message->local_scheduler_id()); - /* Extract the execution dependencies. */ - auto execution_dependencies = - flatbuffers::GetRoot( - message->execution_dependencies()->data()); - /* Extract the task spec. */ - TaskSpec *spec = (TaskSpec *) message->task_spec()->data(); - int64_t task_spec_size = message->task_spec()->size(); - /* Extract the spillback information. */ - int spillback_count = message->spillback_count(); - /* Create a task. */ - /* Allocate the task execution spec on the stack and use it to construct - * the task. - */ - TaskExecutionSpec execution_spec( - from_flatbuf(*execution_dependencies->execution_dependencies()), spec, - task_spec_size, spillback_count); - Task *task = Task_alloc(execution_spec, state, local_scheduler_id); - - /* Call the subscribe callback if there is one. */ - TaskTableSubscribeData *data = - (TaskTableSubscribeData *) callback_data->data->Get(); - if (data->subscribe_callback != NULL) { - data->subscribe_callback(task, data->subscribe_context); - } - Task_free(task); - } else if (strcmp(message_type->str, "subscribe") == 0 || - strcmp(message_type->str, "psubscribe") == 0) { - /* If this condition is true, we got the initial message that acknowledged - * the subscription. */ - if (callback_data->done_callback != NULL) { - task_table_done_callback done_callback = - (task_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, callback_data->user_context); - } - /* Note that we do not destroy the callback data yet because the - * subscription callback needs this data. */ - remove_timer_callback(db->loop, callback_data); - } else { - RAY_LOG(FATAL) << "Unexpected reply type from task table subscribe. " - << "Message type is " << message_type->str; - } -} - -void redis_task_table_subscribe(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - TaskTableSubscribeData *data = - (TaskTableSubscribeData *) callback_data->data->Get(); - /* TASK_CHANNEL_PREFIX is defined in ray_redis_module.cc and must be kept in - * sync with that file. */ - const char *TASK_CHANNEL_PREFIX = "TT:"; - /* In the new code path, subscriptions currently go through the - * primary redis shard. */ - for (auto subscribe_context : db->subscribe_contexts) { - int status; - if (data->local_scheduler_id.is_nil()) { - /* TODO(swang): Implement the state_filter by translating the bitmask into - * a Redis key-matching pattern. */ - status = redisAsyncCommand( - subscribe_context, redis_task_table_subscribe_callback, - (void *) callback_data->timer_id, "PSUBSCRIBE %s*:%d", - TASK_CHANNEL_PREFIX, data->state_filter); - } else { - DBClientID local_scheduler_id = data->local_scheduler_id; - status = redisAsyncCommand( - subscribe_context, redis_task_table_subscribe_callback, - (void *) callback_data->timer_id, "SUBSCRIBE %s%b:%d", - TASK_CHANNEL_PREFIX, (char *) local_scheduler_id.data(), - sizeof(local_scheduler_id), data->state_filter); - } - if ((status == REDIS_ERR) || subscribe_context->err) { - LOG_REDIS_DEBUG(subscribe_context, "error in redis_task_table_subscribe"); - } - } -} - -/* - * ==== db client table callbacks ==== - */ - -void redis_db_client_table_remove_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - - /* Call the done callback if there is one. */ - db_client_table_done_callback done_callback = - (db_client_table_done_callback) callback_data->done_callback; - if (done_callback) { - done_callback(callback_data->id, callback_data->user_context); - } - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_db_client_table_remove(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - int status = - redisAsyncCommand(db->context, redis_db_client_table_remove_callback, - (void *) callback_data->timer_id, "RAY.DISCONNECT %b", - callback_data->id.data(), sizeof(callback_data->id)); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, "error in db_client_table_remove"); - } -} - -void redis_db_client_table_scan(DBHandle *db, - std::vector &db_clients) { - /* TODO(swang): Integrate this functionality with the Ray Redis module. To do - * this, we need the KEYS or SCAN command in Redis modules. */ - /* Get all the database client keys. */ - redisReply *reply = (redisReply *) redisCommand(db->sync_context, "KEYS %s*", - DB_CLIENT_PREFIX); - if (reply->type == REDIS_REPLY_NIL) { - return; - } - /* Get all the database client information. */ - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - for (size_t i = 0; i < reply->elements; ++i) { - /* Strip the database client table prefix. */ - unsigned char *key = (unsigned char *) reply->element[i]->str; - key += strlen(DB_CLIENT_PREFIX); - size_t key_len = reply->element[i]->len; - key_len -= strlen(DB_CLIENT_PREFIX); - /* Get the database client's information. */ - DBClient db_client = redis_db_client_table_get(db, key, key_len); - db_clients.push_back(db_client); - } - freeReplyObject(reply); -} - -void redis_db_client_table_subscribe_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements > 2); - /* First entry is message type, then possibly the regex we psubscribed to, - * then topic, then payload. */ - redisReply *payload = reply->element[reply->elements - 1]; - /* If this condition is true, we got the initial message that acknowledged the - * subscription. */ - if (payload->str == NULL) { - if (callback_data->done_callback) { - db_client_table_done_callback done_callback = - (db_client_table_done_callback) callback_data->done_callback; - done_callback(callback_data->id, callback_data->user_context); - } - /* Note that we do not destroy the callback data yet because the - * subscription callback needs this data. */ - remove_timer_callback(db->loop, callback_data); - - /* Get the current db client table entries, in case we missed notifications - * before the initial subscription. This must be done before we process any - * notifications from the subscription channel, so that we don't readd an - * entry that has already been deleted. */ - std::vector db_clients; - redis_db_client_table_scan(db, db_clients); - /* Call the subscription callback for all entries that we missed. */ - DBClientTableSubscribeData *data = - (DBClientTableSubscribeData *) callback_data->data->Get(); - for (auto db_client : db_clients) { - data->subscribe_callback(&db_client, data->subscribe_context); - } - return; - } - /* Otherwise, parse the payload and call the callback. */ - auto message = - flatbuffers::GetRoot(payload->str); - - /* Parse the client type and auxiliary address from the response. If there is - * only client type, then the update was a delete. */ - DBClient db_client; - db_client.id = from_flatbuf(*message->db_client_id()); - db_client.client_type = std::string(message->client_type()->data()); - db_client.manager_address = std::string(message->manager_address()->data()); - db_client.is_alive = message->is_insertion(); - - /* Call the subscription callback. */ - DBClientTableSubscribeData *data = - (DBClientTableSubscribeData *) callback_data->data->Get(); - if (data->subscribe_callback) { - data->subscribe_callback(&db_client, data->subscribe_context); - } -} - -void redis_db_client_table_subscribe(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - int status = redisAsyncCommand( - db->subscribe_context, redis_db_client_table_subscribe_callback, - (void *) callback_data->timer_id, "SUBSCRIBE db_clients"); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, - "error in db_client_table_register_callback"); - } -} - -void redis_local_scheduler_table_subscribe_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements == 3); - redisReply *message_type = reply->element[0]; - RAY_LOG(DEBUG) << "Local scheduler table subscribe callback, message " - << message_type->str; - - if (strcmp(message_type->str, "message") == 0) { - /* Handle a local scheduler heartbeat. Parse the payload and call the - * subscribe callback. */ - auto message = - flatbuffers::GetRoot(reply->element[2]->str); - - /* Extract the client ID. */ - DBClientID client_id = from_flatbuf(*message->db_client_id()); - /* Extract the fields of the local scheduler info struct. */ - LocalSchedulerInfo info; - if (message->is_dead()) { - /* If the local scheduler is dead, then ignore all other fields in the - * message. */ - info.is_dead = true; - } else { - /* If the local scheduler is alive, collect load information. */ - info.is_dead = false; - info.total_num_workers = message->total_num_workers(); - info.task_queue_length = message->task_queue_length(); - info.available_workers = message->available_workers(); - - info.static_resources = map_from_flatbuf(*message->static_resources()); - info.dynamic_resources = map_from_flatbuf(*message->dynamic_resources()); - } - - /* Call the subscribe callback. */ - LocalSchedulerTableSubscribeData *data = - (LocalSchedulerTableSubscribeData *) callback_data->data->Get(); - if (data->subscribe_callback) { - data->subscribe_callback(client_id, info, data->subscribe_context); - } - } else if (strcmp(message_type->str, "subscribe") == 0) { - /* The reply for the initial SUBSCRIBE command. */ - RAY_CHECK(callback_data->done_callback == NULL); - /* If the initial SUBSCRIBE was successful, clean up the timer, but don't - * destroy the callback data. */ - remove_timer_callback(db->loop, callback_data); - - } else { - RAY_LOG(FATAL) << "Unexpected reply type from local scheduler subscribe."; - } -} - -void redis_local_scheduler_table_subscribe(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - int status = redisAsyncCommand( - db->subscribe_context, redis_local_scheduler_table_subscribe_callback, - (void *) callback_data->timer_id, "SUBSCRIBE local_schedulers"); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, - "error in redis_local_scheduler_table_subscribe"); - } -} - -void redis_local_scheduler_table_send_info_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_INTEGER); - RAY_LOG(DEBUG) << reply->integer << " subscribers received this publish."; - - RAY_CHECK(callback_data->done_callback == NULL); - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_local_scheduler_table_send_info(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - LocalSchedulerTableSendInfoData *data = - (LocalSchedulerTableSendInfoData *) callback_data->data->Get(); - - int64_t size = data->size; - uint8_t *flatbuffer_data = data->flatbuffer_data; - - int status = redisAsyncCommand( - db->context, redis_local_scheduler_table_send_info_callback, - (void *) callback_data->timer_id, "PUBLISH local_schedulers %b", - flatbuffer_data, size); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, - "error in redis_local_scheduler_table_send_info"); - } -} - -void redis_local_scheduler_table_disconnect(DBHandle *db) { - flatbuffers::FlatBufferBuilder fbb; - /* Create the flatbuffers message. */ - std::unordered_map empty_resource_map; - /* Most of the flatbuffer message fields don't matter here. Only the - * db_client_id and the is_dead field matter. */ - auto message = CreateLocalSchedulerInfoMessage( - fbb, to_flatbuf(fbb, db->client), 0, 0, 0, - map_to_flatbuf(fbb, empty_resource_map), - map_to_flatbuf(fbb, empty_resource_map), true); - fbb.Finish(message); - - redisReply *reply = (redisReply *) redisCommand( - db->sync_context, "PUBLISH local_schedulers %b", fbb.GetBufferPointer(), - (size_t) fbb.GetSize()); - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(reply->type == REDIS_REPLY_INTEGER); - RAY_LOG(DEBUG) << reply->integer << " subscribers received this publish."; - freeReplyObject(reply); -} - -void redis_driver_table_subscribe_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements == 3); - redisReply *message_type = reply->element[0]; - RAY_LOG(DEBUG) << "Driver table subscribe callback, message " - << message_type->str; - - if (strcmp(message_type->str, "message") == 0) { - /* Handle a driver heartbeat. Parse the payload and call the subscribe - * callback. */ - auto message = - flatbuffers::GetRoot(reply->element[2]->str); - /* Extract the client ID. */ - WorkerID driver_id = from_flatbuf(*message->driver_id()); - - /* Call the subscribe callback. */ - DriverTableSubscribeData *data = - (DriverTableSubscribeData *) callback_data->data->Get(); - if (data->subscribe_callback) { - data->subscribe_callback(driver_id, data->subscribe_context); - } - } else if (strcmp(message_type->str, "subscribe") == 0) { - /* The reply for the initial SUBSCRIBE command. */ - RAY_CHECK(callback_data->done_callback == NULL); - /* If the initial SUBSCRIBE was successful, clean up the timer, but don't - * destroy the callback data. */ - remove_timer_callback(db->loop, callback_data); - - } else { - RAY_LOG(FATAL) << "Unexpected reply type from driver subscribe."; - } -} - -void redis_driver_table_subscribe(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - int status = redisAsyncCommand( - db->subscribe_context, redis_driver_table_subscribe_callback, - (void *) callback_data->timer_id, "SUBSCRIBE driver_deaths"); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, - "error in redis_driver_table_subscribe"); - } -} - -void redis_driver_table_send_driver_death_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_INTEGER); - RAY_LOG(DEBUG) << reply->integer << " subscribers received this publish."; - /* At the very least, the local scheduler that publishes this message should - * also receive it. */ - RAY_CHECK(reply->integer >= 1); - - RAY_CHECK(callback_data->done_callback == NULL); - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_driver_table_send_driver_death(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - WorkerID driver_id = callback_data->id; - - /* Create a flatbuffer object to serialize and publish. */ - flatbuffers::FlatBufferBuilder fbb; - /* Create the flatbuffers message. */ - auto message = CreateDriverTableMessage(fbb, to_flatbuf(fbb, driver_id)); - fbb.Finish(message); - - int status = redisAsyncCommand( - db->context, redis_driver_table_send_driver_death_callback, - (void *) callback_data->timer_id, "PUBLISH driver_deaths %b", - fbb.GetBufferPointer(), (size_t) fbb.GetSize()); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, - "error in redis_driver_table_send_driver_death"); - } -} - -void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - /* NOTE(swang): We purposefully do not provide a callback, leaving the table - * operation and timer active. This allows us to send a new heartbeat every - * heartbeat_timeout_milliseconds without having to allocate and deallocate - * memory for callback data each time. */ - int status = redisAsyncCommand( - db->context, NULL, (void *) callback_data->timer_id, - "PUBLISH plasma_managers %b", db->client.data(), sizeof(db->client)); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, - "error in redis_plasma_manager_send_heartbeat"); - } - /* Clean up the timer and callback. */ - destroy_timer_callback(db->loop, callback_data); -} - -void redis_publish_actor_creation_notification_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_INTEGER); - RAY_LOG(DEBUG) << reply->integer << " subscribers received this publish."; - // At the very least, the local scheduler that publishes this message should - // also receive it. - RAY_CHECK(reply->integer >= 1); - - RAY_CHECK(callback_data->done_callback == NULL); - // Clean up the timer and callback. - destroy_timer_callback(db->loop, callback_data); -} - -void redis_publish_actor_creation_notification( - TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - - ActorCreationNotificationData *data = - (ActorCreationNotificationData *) callback_data->data->Get(); - - int status = redisAsyncCommand( - db->context, redis_publish_actor_creation_notification_callback, - (void *) callback_data->timer_id, "PUBLISH actor_notifications %b", - &data->flatbuffer_data[0], data->size); - if ((status == REDIS_ERR) || db->context->err) { - LOG_REDIS_DEBUG(db->context, - "error in redis_publish_actor_creation_notification"); - } -} - -void redis_actor_notification_table_subscribe_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - - redisReply *reply = (redisReply *) r; - RAY_CHECK(reply->type == REDIS_REPLY_ARRAY); - RAY_CHECK(reply->elements == 3); - redisReply *message_type = reply->element[0]; - RAY_LOG(DEBUG) << "Local scheduler table subscribe callback, message " - << message_type->str; - - if (strcmp(message_type->str, "message") == 0) { - // Handle an actor notification message. Parse the payload and call the - // subscribe callback. - redisReply *payload = reply->element[2]; - ActorNotificationTableSubscribeData *data = - (ActorNotificationTableSubscribeData *) callback_data->data->Get(); - - auto message = - flatbuffers::GetRoot(payload->str); - ActorID actor_id = from_flatbuf(*message->actor_id()); - WorkerID driver_id = from_flatbuf(*message->driver_id()); - DBClientID local_scheduler_id = - from_flatbuf(*message->local_scheduler_id()); - - if (data->subscribe_callback) { - data->subscribe_callback(actor_id, driver_id, local_scheduler_id, - data->subscribe_context); - } - } else if (strcmp(message_type->str, "subscribe") == 0) { - /* The reply for the initial SUBSCRIBE command. */ - RAY_CHECK(callback_data->done_callback == NULL); - /* If the initial SUBSCRIBE was successful, clean up the timer, but don't - * destroy the callback data. */ - remove_timer_callback(db->loop, callback_data); - - } else { - RAY_LOG(FATAL) << "Unexpected reply type from actor notification " - << "subscribe."; - } -} - -void redis_actor_notification_table_subscribe( - TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - int status = redisAsyncCommand( - db->subscribe_context, redis_actor_notification_table_subscribe_callback, - (void *) callback_data->timer_id, "SUBSCRIBE actor_notifications"); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, - "error in redis_actor_notification_table_subscribe"); - } -} - -void redis_actor_table_mark_removed(DBHandle *db, ActorID actor_id) { - int status = - redisAsyncCommand(db->context, NULL, NULL, "HSET Actor:%b removed \"1\"", - actor_id.data(), sizeof(actor_id)); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->context, "error in redis_actor_table_mark_removed"); - } -} - -void redis_push_error_rpush_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - /* The reply should be the length of the errors list after our RPUSH. */ - RAY_CHECK(reply->type == REDIS_REPLY_INTEGER); - destroy_timer_callback(db->loop, callback_data); -} - -void redis_push_error_hmset_callback(redisAsyncContext *c, - void *r, - void *privdata) { - REDIS_CALLBACK_HEADER(db, callback_data, r); - redisReply *reply = (redisReply *) r; - - /* Make sure we were able to add the error information. */ - RAY_CHECK(reply->type != REDIS_REPLY_ERROR) << "reply->str is " << reply->str; - RAY_CHECK(strcmp(reply->str, "OK") == 0) << "reply->str is " << reply->str; - - /* Add the error to this driver's list of errors. */ - ErrorInfo *info = (ErrorInfo *) callback_data->data->Get(); - int status = redisAsyncCommand( - db->context, redis_push_error_rpush_callback, - (void *) callback_data->timer_id, "RPUSH ErrorKeys Error:%b:%b", - info->driver_id.data(), sizeof(info->driver_id), info->error_key.data(), - sizeof(info->error_key)); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error rpush"); - } -} - -void redis_push_error(TableCallbackData *callback_data) { - DBHandle *db = callback_data->db_handle; - ErrorInfo *info = (ErrorInfo *) callback_data->data->Get(); - RAY_CHECK(info->error_type < ErrorIndex::MAX && - info->error_type >= ErrorIndex::OBJECT_HASH_MISMATCH); - /// Look up the error type. - const char *error_type = error_types[static_cast(info->error_type)]; - - /* Set the error information. */ - int status = redisAsyncCommand( - db->context, redis_push_error_hmset_callback, - (void *) callback_data->timer_id, - "HMSET Error:%b:%b type %s message %b data %b", info->driver_id.data(), - sizeof(info->driver_id), info->error_key.data(), sizeof(info->error_key), - error_type, info->error_message, info->size, "None", strlen("None")); - if ((status == REDIS_ERR) || db->subscribe_context->err) { - LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error hmset"); - } -} - -DBClientID get_db_client_id(DBHandle *db) { - RAY_CHECK(db != NULL); - return db->client; -} diff --git a/src/common/state/redis.h b/src/common/state/redis.h deleted file mode 100644 index 164069740..000000000 --- a/src/common/state/redis.h +++ /dev/null @@ -1,356 +0,0 @@ -#ifndef REDIS_H -#define REDIS_H - -#include - -#include "db.h" -#include "db_client_table.h" -#include "object_table.h" -#include "task_table.h" - -#include "hiredis/hiredis.h" -#include "hiredis/async.h" - -#define LOG_REDIS_ERROR(context, M, ...) \ - RAY_LOG(ERROR) << "Redis error " << context->err << " " << context->errstr \ - << "; " << M - -#define LOG_REDIS_DEBUG(context, M, ...) \ - RAY_LOG(DEBUG) << "Redis error " << context->err << " " << context->errstr \ - << "; " << M; - -struct DBHandle { - /** String that identifies this client type. */ - char *client_type; - /** Unique ID for this client. */ - DBClientID client; - /** Primary redis context for all non-subscribe connections. This is used for - * the database client table, heartbeats, and errors that should be pushed to - * the driver. */ - redisAsyncContext *context; - /** Primary redis context for "subscribe" communication. A separate context - * is needed for this communication (see - * https://github.com/redis/hiredis/issues/55). This is used for the - * database client table, heartbeats, and errors that should be pushed to - * the driver. */ - redisAsyncContext *subscribe_context; - /** Redis contexts for shards for all non-subscribe connections. All requests - * to the object table, task table, and event table should be directed here. - * The correct shard can be retrieved using get_redis_context below. */ - std::vector contexts; - /** Redis contexts for shards for "subscribe" communication. All requests - * to the object table, task table, and event table should be directed here. - * The correct shard can be retrieved using get_redis_context below. */ - std::vector subscribe_contexts; - /** The event loop this global state store connection is part of. */ - event_loop *loop; - /** Index of the database connection in the event loop */ - int64_t db_index; - /** Cache for the IP addresses of db clients. This is an unordered map mapping - * client IDs to addresses. */ - std::unordered_map db_client_cache; - /** Redis context for synchronous connections. This should only be used very - * rarely, it is not asynchronous. */ - redisContext *sync_context; -}; - -/** - * Get the Redis asynchronous context responsible for non-subscription - * communication for the given UniqueID. - * - * @param db The database handle. - * @param id The ID whose location we are querying for. - * @return The redisAsyncContext responsible for the given ID. - */ -redisAsyncContext *get_redis_context(DBHandle *db, UniqueID id); - -/** - * Get the Redis asynchronous context responsible for subscription - * communication for the given UniqueID. - * - * @param db The database handle. - * @param id The ID whose location we are querying for. - * @return The redisAsyncContext responsible for the given ID. - */ -redisAsyncContext *get_redis_subscribe_context(DBHandle *db, UniqueID id); - -/** - * Get a list of Redis shard IP addresses from the primary shard. - * - * @param context A Redis context connected to the primary shard. - * @param db_shards_addresses The IP addresses for the shards registered - * with the primary shard will be added to this vector. - * @param db_shards_ports The IP ports for the shards registered with the - * primary shard will be added to this vector, in the same order as - * db_shards_addresses. - */ -void get_redis_shards(redisContext *context, - std::vector &db_shards_addresses, - std::vector &db_shards_ports); - -void redis_cache_set_db_client(DBHandle *db, DBClient client); - -DBClient redis_cache_get_db_client(DBHandle *db, DBClientID db_client_id); - -void redis_object_table_get_entry(redisAsyncContext *c, - void *r, - void *privdata); - -void object_table_lookup_callback(redisAsyncContext *c, - void *r, - void *privdata); - -/* - * ==== Redis object table functions ==== - */ - -/** - * Lookup object table entry in redis. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_object_table_lookup(TableCallbackData *callback_data); - -/** - * Add a location entry to the object table in redis. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_object_table_add(TableCallbackData *callback_data); - -/** - * Remove a location entry from the object table in redis. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_object_table_remove(TableCallbackData *callback_data); - -/** - * Create a client-specific channel for receiving notifications from the object - * table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_object_table_subscribe_to_notifications( - TableCallbackData *callback_data); - -/** - * Request notifications about when certain objects become available. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_object_table_request_notifications(TableCallbackData *callback_data); - -/** - * Add a new object to the object table in redis. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_result_table_add(TableCallbackData *callback_data); - -/** - * Lookup the task that created the object in redis. The result is the task ID. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_result_table_lookup(TableCallbackData *callback_data); - -/** - * Callback invoked when the reply from the object table lookup command is - * received. - * - * @param c Redis context. - * @param r Reply. - * @param privdata Data associated to the callback. - * @return Void. - */ -void redis_object_table_lookup_callback(redisAsyncContext *c, - void *r, - void *privdata); - -/* - * ==== Redis task table function ===== - */ - -/** - * Get a task table entry, including the task spec and the task's scheduling - * information. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_task_table_get_task(TableCallbackData *callback_data); - -/** - * Add a task table entry with a new task spec and the task's scheduling - * information. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_task_table_add_task(TableCallbackData *callback_data); - -/** - * Update a task table entry with the task's scheduling information. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_task_table_update(TableCallbackData *callback_data); - -/** - * Update a task table entry with the task's scheduling information, if the - * task's current scheduling information matches the test value. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_task_table_test_and_update(TableCallbackData *callback_data); - -/** - * Callback invoked when the reply from the task push command is received. - * - * @param c Redis context. - * @param r Reply (not used). - * @param privdata Data associated to the callback. - * @return Void. - */ -void redis_task_table_publish_push_callback(redisAsyncContext *c, - void *r, - void *privdata); - -/** - * Callback invoked when the reply from the task publish command is received. - * - * @param c Redis context. - * @param r Reply (not used). - * @param privdata Data associated to the callback. - * @return Void. - */ -void redis_task_table_publish_publish_callback(redisAsyncContext *c, - void *r, - void *privdata); - -/** - * Subscribe to updates of the task table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_task_table_subscribe(TableCallbackData *callback_data); - -/** - * Remove a client from the db clients table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_db_client_table_remove(TableCallbackData *callback_data); - -/** - * Subscribe to updates from the db client table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_db_client_table_subscribe(TableCallbackData *callback_data); - -/** - * Subscribe to updates from the local scheduler table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_local_scheduler_table_subscribe(TableCallbackData *callback_data); - -/** - * Publish an update to the local scheduler table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_local_scheduler_table_send_info(TableCallbackData *callback_data); - -/** - * Synchronously publish a null update to the local scheduler table signifying - * that we are about to exit. - * - * @param db The database handle of the dying local scheduler. - * @return Void. - */ -void redis_local_scheduler_table_disconnect(DBHandle *db); - -/** - * Subscribe to updates from the driver table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_driver_table_subscribe(TableCallbackData *callback_data); - -/** - * Publish an update to the driver table. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_driver_table_send_driver_death(TableCallbackData *callback_data); - -void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data); - -/** - * Marks an actor as removed. This prevents the actor from being resurrected. - * - * @param db The database handle. - * @param actor_id The actor id to mark as removed. - * @return Void. - */ -void redis_actor_table_mark_removed(DBHandle *db, ActorID actor_id); - -/// Publish an actor creation notification. -/// -/// \param callback_data Data structure containing redis connection and timeout -/// information. -/// \return Void. -void redis_publish_actor_creation_notification( - TableCallbackData *callback_data); - -/** - * Subscribe to updates about newly created actors. - * - * @param callback_data Data structure containing redis connection and timeout - * information. - * @return Void. - */ -void redis_actor_notification_table_subscribe(TableCallbackData *callback_data); - -void redis_object_info_subscribe(TableCallbackData *callback_data); - -void redis_push_error(TableCallbackData *callback_data); - -#endif /* REDIS_H */ diff --git a/src/common/state/table.cc b/src/common/state/table.cc deleted file mode 100644 index 8269c2b1e..000000000 --- a/src/common/state/table.cc +++ /dev/null @@ -1,200 +0,0 @@ -#include "table.h" - -#include -#include -#include "redis.h" - -BaseCallbackData::BaseCallbackData(void *data) { - data_ = data; -} - -BaseCallbackData::~BaseCallbackData(void) {} - -void *BaseCallbackData::Get(void) { - return data_; -} - -CommonCallbackData::CommonCallbackData(void *data) : BaseCallbackData(data) {} - -CommonCallbackData::~CommonCallbackData(void) { - free(data_); -} - -TaskCallbackData::TaskCallbackData(Task *task_data) - : BaseCallbackData(task_data) {} - -TaskCallbackData::~TaskCallbackData(void) { - Task *task = (Task *) data_; - Task_free(task); -} - -/* The default behavior is to retry every ten seconds forever. */ -static const RetryInfo default_retry = {.num_retries = -1, - .timeout = 10000, - .fail_callback = NULL}; - -static int64_t callback_data_id = 0; - -TableCallbackData *init_table_callback(DBHandle *db_handle, - UniqueID id, - const char *label, - OWNER BaseCallbackData *data, - RetryInfo *retry, - table_done_callback done_callback, - table_retry_callback retry_callback, - void *user_context) { - RAY_CHECK(db_handle); - RAY_CHECK(db_handle->loop); - RAY_CHECK(data); - /* If no retry info is provided, use the default retry info. */ - if (retry == NULL) { - retry = (RetryInfo *) &default_retry; - } - RAY_CHECK(retry); - /* Allocate and initialize callback data structure for object table */ - TableCallbackData *callback_data = - (TableCallbackData *) malloc(sizeof(TableCallbackData)); - RAY_CHECK(callback_data != NULL) << "Memory allocation error!"; - callback_data->id = id; - callback_data->label = label; - callback_data->retry = *retry; - callback_data->done_callback = done_callback; - callback_data->retry_callback = retry_callback; - callback_data->data = data; - callback_data->requests_info = NULL; - callback_data->user_context = user_context; - callback_data->db_handle = db_handle; - /* TODO(ekl) set a retry timer once we've figured out the retry conditions - * and have a solution to the O(n^2) ae timers issue. For now, use a dummy - * timer id to uniquely id this callback. */ - callback_data->timer_id = callback_data_id++; - outstanding_callbacks_add(callback_data); - - RAY_LOG(DEBUG) << "Initializing table command " << callback_data->label - << " with timer ID " << callback_data->timer_id; - callback_data->retry_callback(callback_data); - - return callback_data; -} - -void destroy_timer_callback(event_loop *loop, - TableCallbackData *callback_data) { - /* This is commented out because we no longer add timers to the event loop for - * each Redis command. */ - // event_loop_remove_timer(loop, callback_data->timer_id); - destroy_table_callback(callback_data); -} - -void remove_timer_callback(event_loop *loop, TableCallbackData *callback_data) { - /* This is commented out because we no longer add timers to the event loop for - * each Redis command. */ - // event_loop_remove_timer(loop, callback_data->timer_id); -} - -void destroy_table_callback(TableCallbackData *callback_data) { - RAY_CHECK(callback_data != NULL); - - if (callback_data->requests_info) - free(callback_data->requests_info); - - RAY_CHECK(callback_data->data != NULL); - delete callback_data->data; - callback_data->data = NULL; - - outstanding_callbacks_remove(callback_data); - - /* Timer is removed via EVENT_LOOP_TIMER_DONE in the timeout callback. */ - free(callback_data); -} - -int64_t table_timeout_handler(event_loop *loop, - int64_t timer_id, - void *user_context) { - RAY_CHECK(loop != NULL); - RAY_CHECK(user_context != NULL); - TableCallbackData *callback_data = (TableCallbackData *) user_context; - - RAY_CHECK(callback_data->retry.num_retries >= 0 || - callback_data->retry.num_retries == -1); - RAY_LOG(WARNING) << "retrying operation " << callback_data->label - << ", retry_count = " << callback_data->retry.num_retries; - - if (callback_data->retry.num_retries == 0) { - /* We didn't get a response from the database after exhausting all retries; - * let user know, cleanup the state, and remove the timer. */ - RAY_LOG(WARNING) << "Table command " << callback_data->label - << " with timer ID " << timer_id << " failed"; - if (callback_data->retry.fail_callback) { - callback_data->retry.fail_callback(callback_data->id, - callback_data->user_context, - callback_data->data->Get()); - } - destroy_table_callback(callback_data); - return EVENT_LOOP_TIMER_DONE; - } - - /* Decrement retry count and try again. We use -1 to indicate infinite - * retries. */ - if (callback_data->retry.num_retries != -1) { - callback_data->retry.num_retries--; - } - callback_data->retry_callback(callback_data); - return callback_data->retry.timeout; -} - -/** - * Unordered map maintaining the outstanding callbacks. - * - * This unordered map is used to handle the following case: - * - a table command is issued with an associated callback and a callback data - * structure; - * - the last timeout associated to this command expires, as a result the - * callback data structure is freed; - * - a reply arrives, but now the callback data structure is gone, so we have - * to ignore this reply; - * - * This unordered map enables us to ignore such replies. The operations on the - * unordered map are as follows. - * - * When we issue a table command and a timeout event to wait for the reply, we - * add a new entry to the unordered map that is keyed by the ID of the timer. - * Note that table commands must have unique timer IDs, which are assigned by - * the Redis ae event loop. - * - * When we receive the reply, we check whether the callback still exists in - * this unordered map, and if not we just ignore the reply. If the callback does - * exist, the reply receiver is responsible for removing the timer and the - * entry associated to the callback, or else the timeout handler will continue - * firing. - * - * When the last timeout associated to the command expires we remove the entry - * associated to the callback. - */ -static std::unordered_map outstanding_callbacks; - -void outstanding_callbacks_add(TableCallbackData *callback_data) { - outstanding_callbacks[callback_data->timer_id] = callback_data; -} - -TableCallbackData *outstanding_callbacks_find(int64_t key) { - auto it = outstanding_callbacks.find(key); - if (it != outstanding_callbacks.end()) { - return it->second; - } - return NULL; -} - -void outstanding_callbacks_remove(TableCallbackData *callback_data) { - outstanding_callbacks.erase(callback_data->timer_id); -} - -void destroy_outstanding_callbacks(event_loop *loop) { - /* We have to be careful because destroy_timer_callback modifies - * outstanding_callbacks in place */ - auto it = outstanding_callbacks.begin(); - while (it != outstanding_callbacks.end()) { - auto next_it = std::next(it, 1); - destroy_timer_callback(loop, it->second); - it = next_it; - } -} diff --git a/src/common/state/table.h b/src/common/state/table.h deleted file mode 100644 index 1fadcf339..000000000 --- a/src/common/state/table.h +++ /dev/null @@ -1,216 +0,0 @@ -#ifndef TABLE_H -#define TABLE_H - -#include "common.h" -#include "task.h" -#include "db.h" - -typedef struct TableCallbackData TableCallbackData; - -/* An abstract class for any data passed by the user into a table operation. - * This class wraps arbitrary pointers and allows the caller to define a custom - * destructor, for data that is not allocated with malloc. */ -class BaseCallbackData { - public: - BaseCallbackData(void *data); - virtual ~BaseCallbackData(void) = 0; - - /* Return the pointer to the data. */ - void *Get(void); - - protected: - /* The pointer to the data. */ - void *data_; -}; - -/* A common class for malloc'ed data passed by the user into a table operation. - * This should ONLY be used when only a free is necessary. */ -class CommonCallbackData : public BaseCallbackData { - public: - CommonCallbackData(void *data); - ~CommonCallbackData(void); -}; - -/* A class for Task data passed by the user into a table operation. This calls - * task cleanup in the destructor. */ -class TaskCallbackData : public BaseCallbackData { - public: - TaskCallbackData(Task *task_data); - ~TaskCallbackData(void); -}; - -typedef void *table_done_callback; - -/* The callback called when the database operation hasn't completed after - * the number of retries specified for the operation. - * - * @param id The unique ID that identifies this callback. Examples include an - * object ID or task ID. - * @param user_context The state context for the callback. This is equivalent - * to the user_context field in TableCallbackData. - * @param user_data A data argument for the callback. This is equivalent to the - * data field in TableCallbackData. The user is responsible for - * freeing user_data. - */ -typedef void (*table_fail_callback)(UniqueID id, - void *user_context, - void *user_data); - -typedef void (*table_retry_callback)(TableCallbackData *callback_data); - -/** - * Data structure consolidating the retry related variables. If a NULL - * RetryInfo struct is used, the default behavior will be to retry infinitely - * many times. - */ -typedef struct { - /** Number of retries. This field will be decremented every time a retry - * occurs (unless the value is -1). If this value is -1, then there will be - * infinitely many retries. */ - int num_retries; - /** Timeout, in milliseconds. */ - uint64_t timeout; - /** The callback that will be called if there are no more retries left. */ - table_fail_callback fail_callback; -} RetryInfo; - -struct TableCallbackData { - /** ID of the entry in the table that we are going to look up, remove or add. - */ - UniqueID id; - /** A label to identify the original request for logging purposes. */ - const char *label; - /** The callback that will be called when results is returned. */ - table_done_callback done_callback; - /** The callback that will be called to initiate the next try. */ - table_retry_callback retry_callback; - /** Retry information containing the remaining number of retries, the timeout - * before the next retry, and a pointer to the failure callback. - */ - RetryInfo retry; - /** Pointer to the data that is entered into the table. This can be used to - * pass the result of the call to the callback. The callback takes ownership - * over this data and will free it. */ - BaseCallbackData *data; - /** Pointer to the data used internally to handle multiple database requests. - */ - void *requests_info; - /** User context. */ - void *user_context; - /** Handle to db. */ - DBHandle *db_handle; - /** Handle to timer. */ - int64_t timer_id; -}; - -/** - * Function to handle the timeout event. - * - * @param loop Event loop. - * @param timer_id Timer identifier. - * @param context Pointer to the callback data for the object table - * @return Timeout to reset the timer if we need to try again, or - * EVENT_LOOP_TIMER_DONE if retry_count == 0. - */ -int64_t table_timeout_handler(event_loop *loop, - int64_t timer_id, - void *context); - -/** - * Initialize the table callback and call the retry_callback for the first time. - * - * @param db_handle Database handle. - * @param id ID of the object that is looked up, added or removed. - * @param label A string label to identify the type of table request for - * logging purposes. - * @param data Data entered into the table. Shall be freed by the user. Caller - * must specify a destructor by wrapping a void *pointer in a - * BaseCallbackData class. - * @param retry Retry relevant information: retry timeout, number of remaining - * retries, and retry callback. - * @param done_callback Function to be called when database returns result. - * @param fail_callback Function to be called when number of retries is - * exhausted. - * @param user_context Context that can be provided by the user and will be - * passed on to the various callbacks. - * @return New table callback data struct. - */ -TableCallbackData *init_table_callback(DBHandle *db_handle, - UniqueID id, - const char *label, - OWNER BaseCallbackData *data, - RetryInfo *retry, - table_done_callback done_callback, - table_retry_callback retry_callback, - void *user_context); - -/** - * Destroy any state associated with the callback data. This removes all - * associated state from the outstanding callbacks unordered map and frees any - * associated memory. This does not remove any associated timer events. - * - * @param callback_data The pointer to the data structure of the callback we - * want to remove. - * @return Void. - */ -void destroy_table_callback(TableCallbackData *callback_data); - -/** - * Destroy all state events associated with the callback data, including memory - * and timer events. - * - * @param loop The event loop. - * @param callback_data The pointer to the data structure of the callback we - * want to remove. - * @return Void. - */ -void destroy_timer_callback(event_loop *loop, TableCallbackData *callback_data); - -/** - * Remove the callback timer without destroying the callback data. - * - * @param loop The event loop. - * @param callback_data The pointer to the data structure of the callback. - * @return Void. - */ -void remove_timer_callback(event_loop *loop, TableCallbackData *callback_data); - -/** - * Add an outstanding callback entry. - * - * @param callback_data The pointer to the data structure of the callback we - * want to insert. - * @return None. - */ -void outstanding_callbacks_add(TableCallbackData *callback_data); - -/** - * Find an outstanding callback entry. - * - * @param key The key for the outstanding callbacks unordered map. We use the - * timer ID assigned by the Redis ae event loop. - * @return Returns the callback data if found, NULL otherwise. - */ -TableCallbackData *outstanding_callbacks_find(int64_t key); - -/** - * Remove an outstanding callback entry. This only removes the callback entry - * from the unordered map. It does not free the entry or remove any associated - * timer events. - * - * @param callback_data The pointer to the data structure of the callback we - * want to remove. - * @return Void. - */ -void outstanding_callbacks_remove(TableCallbackData *callback_data); - -/** - * Destroy all outstanding callbacks and remove their associated timer events - * from the event loop. - * - * @param loop The event loop from which we want to remove the timer events. - * @return Void. - */ -void destroy_outstanding_callbacks(event_loop *loop); - -#endif /* TABLE_H */ diff --git a/src/common/state/task_table.cc b/src/common/state/task_table.cc deleted file mode 100644 index 514350b08..000000000 --- a/src/common/state/task_table.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "task_table.h" -#include "redis.h" - -#define NUM_DB_REQUESTS 2 - -void task_table_get_task(DBHandle *db_handle, - TaskID task_id, - RetryInfo *retry, - task_table_get_callback get_callback, - void *user_context) { - init_table_callback( - db_handle, task_id, __func__, new CommonCallbackData(NULL), retry, - (void *) get_callback, redis_task_table_get_task, user_context); -} - -void task_table_add_task(DBHandle *db_handle, - OWNER Task *task, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context) { - init_table_callback(db_handle, Task_task_id(task), __func__, - new TaskCallbackData(task), retry, - (table_done_callback) done_callback, - redis_task_table_add_task, user_context); -} - -void task_table_update(DBHandle *db_handle, - OWNER Task *task, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context) { - init_table_callback(db_handle, Task_task_id(task), __func__, - new TaskCallbackData(task), retry, - (table_done_callback) done_callback, - redis_task_table_update, user_context); -} - -void task_table_test_and_update( - DBHandle *db_handle, - TaskID task_id, - DBClientID test_local_scheduler_id, - TaskStatus test_state_bitmask, - TaskStatus update_state, - RetryInfo *retry, - task_table_test_and_update_callback done_callback, - void *user_context) { - TaskTableTestAndUpdateData *update_data = - (TaskTableTestAndUpdateData *) malloc(sizeof(TaskTableTestAndUpdateData)); - update_data->test_local_scheduler_id = test_local_scheduler_id; - update_data->test_state_bitmask = test_state_bitmask; - update_data->update_state = update_state; - /* Update the task entry's local scheduler with this client's ID. */ - update_data->local_scheduler_id = db_handle->client; - init_table_callback(db_handle, task_id, __func__, - new CommonCallbackData(update_data), retry, - (table_done_callback) done_callback, - redis_task_table_test_and_update, user_context); -} - -/* TODO(swang): A corresponding task_table_unsubscribe. */ -void task_table_subscribe(DBHandle *db_handle, - DBClientID local_scheduler_id, - TaskStatus state_filter, - task_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context) { - TaskTableSubscribeData *sub_data = - (TaskTableSubscribeData *) malloc(sizeof(TaskTableSubscribeData)); - sub_data->local_scheduler_id = local_scheduler_id; - sub_data->state_filter = state_filter; - sub_data->subscribe_callback = subscribe_callback; - sub_data->subscribe_context = subscribe_context; - - init_table_callback(db_handle, local_scheduler_id, __func__, - new CommonCallbackData(sub_data), retry, - (table_done_callback) done_callback, - redis_task_table_subscribe, user_context); -} diff --git a/src/common/state/task_table.h b/src/common/state/task_table.h deleted file mode 100644 index 3884ddece..000000000 --- a/src/common/state/task_table.h +++ /dev/null @@ -1,190 +0,0 @@ -#ifndef task_table_H -#define task_table_H - -#include "db.h" -#include "table.h" -#include "task.h" - -/** - * The task table is a message bus that is used for communication between local - * and global schedulers (and also persisted to the state database). Here are - * examples of events that are recorded by the task table: - * - * 1) Local schedulers write to it when submitting a task to the global - * scheduler. - * 2) The global scheduler subscribes to updates to the task table to get tasks - * submitted by local schedulers. - * 3) The global scheduler writes to it when assigning a task to a local - * scheduler. - * 4) Local schedulers subscribe to updates to the task table to get tasks - * assigned to them by the global scheduler. - * 5) Local schedulers write to it when a task finishes execution. - */ - -/* Callback called when a task table write operation completes. */ -typedef void (*task_table_done_callback)(TaskID task_id, void *user_context); - -/* Callback called when a task table read operation completes. If the task ID - * was not in the task table, then the task pointer will be NULL. */ -typedef void (*task_table_get_callback)(Task *task, void *user_context); - -/* Callback called when a task table test-and-update operation completes. If - * the task ID was not in the task table, then the task pointer will be NULL. - * If the update succeeded, the updated field will be set to true. */ -typedef void (*task_table_test_and_update_callback)(Task *task, - void *user_context, - bool updated); - -/** - * Get a task's entry from the task table. - * - * @param db_handle Database handle. - * @param task_id The ID of the task we want to look up. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void task_table_get_task(DBHandle *db, - TaskID task_id, - RetryInfo *retry, - task_table_get_callback get_callback, - void *user_context); - -/** - * Add a task entry, including task spec and scheduling information, to the task - * table. This will overwrite any task already in the task table with the same - * task ID. - * - * @param db_handle Database handle. - * @param task The task entry to add to the table. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void task_table_add_task(DBHandle *db_handle, - OWNER Task *task, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context); - -/* - * ==== Publish the task table ==== - */ - -/** - * Update a task's scheduling information in the task table. This assumes that - * the task spec already exists in the task table entry. - * - * @param db_handle Database handle. - * @param task The task entry to add to the table. The task spec in the entry is - * ignored. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void task_table_update(DBHandle *db_handle, - OWNER Task *task, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context); - -/** - * Update a task's scheduling information in the task table, if the current - * value matches the given test value. If the update succeeds, it also updates - * the task entry's local scheduler ID with the ID of the client who called - * this function. This assumes that the task spec already exists in the task - * table entry. - * - * @param db_handle Database handle. - * @param task_id The task ID of the task entry to update. - * @param test_local_scheduler_id The local scheduler ID to test the current - * local scheduler ID against. If not NIL_ID, and if the current local - * scheduler ID does not match it, then the update will not happen. - * @param test_state_bitmask The bitmask to apply to the task entry's current - * scheduling state. The update happens if and only if the current - * scheduling state AND-ed with the bitmask is greater than 0 and the - * local scheduler ID test passes. - * @param update_state The value to update the task entry's scheduling state - * with, if the current state matches test_state_bitmask. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void task_table_test_and_update( - DBHandle *db_handle, - TaskID task_id, - DBClientID test_local_scheduler_id, - TaskStatus test_state_bitmask, - TaskStatus update_state, - RetryInfo *retry, - task_table_test_and_update_callback done_callback, - void *user_context); - -/* Data that is needed to test and set the task's scheduling state. */ -typedef struct { - /** The value to test the current local scheduler ID against. This field is - * ignored if equal to NIL_ID. */ - DBClientID test_local_scheduler_id; - TaskStatus test_state_bitmask; - TaskStatus update_state; - DBClientID local_scheduler_id; -} TaskTableTestAndUpdateData; - -/* - * ==== Subscribing to the task table ==== - */ - -/* Callback for subscribing to the task table. */ -typedef void (*task_table_subscribe_callback)(Task *task, void *user_context); - -/** - * Register a callback for a task event. An event is any update of a task in - * the task table, produced by task_table_add_task or task_table_add_task. - * Events include changes to the task's scheduling state or changes to the - * task's local scheduler ID. - * - * @param db_handle Database handle. - * @param subscribe_callback Callback that will be called when the task table is - * updated. - * @param subscribe_context Context that will be passed into the - * subscribe_callback. - * @param local_scheduler_id The db_client_id of the local scheduler whose - * events we want to listen to. If you want to subscribe to updates from - * all local schedulers, pass in NIL_ID. - * @param state_filter Events we want to listen to. Can have values from the - * enum "scheduling_state" in task.h. - * TODO(pcm): Make it possible to combine these using flags like - * TASK_STATUS_WAITING | TASK_STATUS_SCHEDULED. - * @param retry Information about retrying the request to the database. - * @param done_callback Function to be called when database returns result. - * @param user_context Data that will be passed to done_callback and - * fail_callback. - * @return Void. - */ -void task_table_subscribe(DBHandle *db_handle, - DBClientID local_scheduler_id, - TaskStatus state_filter, - task_table_subscribe_callback subscribe_callback, - void *subscribe_context, - RetryInfo *retry, - task_table_done_callback done_callback, - void *user_context); - -/* Data that is needed to register task table subscribe callbacks with the state - * database. */ -typedef struct { - DBClientID local_scheduler_id; - TaskStatus state_filter; - task_table_subscribe_callback subscribe_callback; - void *subscribe_context; -} TaskTableSubscribeData; - -#endif /* task_table_H */ diff --git a/src/common/task.cc b/src/common/task.cc deleted file mode 100644 index 60110fe22..000000000 --- a/src/common/task.cc +++ /dev/null @@ -1,606 +0,0 @@ -#include - -#include "common_protocol.h" - -#include "task.h" - -extern "C" { -#include "sha256.h" -} - -ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) { - /* Here, return_indices need to be >= 0, so we can use negative - * indices for put. */ - RAY_DCHECK(return_index >= 0); - /* TODO(rkn): This line requires object and task IDs to be the same size. */ - ObjectID return_id = task_id; - int64_t *first_bytes = (int64_t *) &return_id; - /* XOR the first bytes of the object ID with the return index. We add one so - * the first return ID is not the same as the task ID. */ - *first_bytes = *first_bytes ^ (return_index + 1); - return return_id; -} - -ObjectID task_compute_put_id(TaskID task_id, int64_t put_index) { - RAY_DCHECK(put_index >= 0); - /* TODO(pcm): This line requires object and task IDs to be the same size. */ - ObjectID put_id = task_id; - int64_t *first_bytes = (int64_t *) &put_id; - /* XOR the first bytes of the object ID with the return index. We add one so - * the first return ID is not the same as the task ID. */ - *first_bytes = *first_bytes ^ (-put_index - 1); - return put_id; -} - -class TaskBuilder { - public: - void Start(UniqueID driver_id, - TaskID parent_task_id, - int64_t parent_counter, - ActorID actor_creation_id, - ObjectID actor_creation_dummy_object_id, - ActorID actor_id, - ActorHandleID actor_handle_id, - int64_t actor_counter, - bool is_actor_checkpoint_method, - FunctionID function_id, - int64_t num_returns) { - driver_id_ = driver_id; - parent_task_id_ = parent_task_id; - parent_counter_ = parent_counter; - actor_creation_id_ = actor_creation_id; - actor_creation_dummy_object_id_ = actor_creation_dummy_object_id; - actor_id_ = actor_id; - actor_handle_id_ = actor_handle_id; - actor_counter_ = actor_counter; - is_actor_checkpoint_method_ = is_actor_checkpoint_method; - function_id_ = function_id; - num_returns_ = num_returns; - - /* Compute hashes. */ - sha256_init(&ctx); - sha256_update(&ctx, (BYTE *) &driver_id, sizeof(driver_id)); - sha256_update(&ctx, (BYTE *) &parent_task_id, sizeof(parent_task_id)); - sha256_update(&ctx, (BYTE *) &parent_counter, sizeof(parent_counter)); - sha256_update(&ctx, (BYTE *) &actor_creation_id, sizeof(actor_creation_id)); - sha256_update(&ctx, (BYTE *) &actor_creation_dummy_object_id, - sizeof(actor_creation_dummy_object_id)); - sha256_update(&ctx, (BYTE *) &actor_id, sizeof(actor_id)); - sha256_update(&ctx, (BYTE *) &actor_counter, sizeof(actor_counter)); - sha256_update(&ctx, (BYTE *) &is_actor_checkpoint_method, - sizeof(is_actor_checkpoint_method)); - sha256_update(&ctx, (BYTE *) &function_id, sizeof(function_id)); - } - - void NextReferenceArgument(ObjectID object_ids[], int num_object_ids) { - args.push_back( - CreateArg(fbb, to_flatbuf(fbb, &object_ids[0], num_object_ids))); - sha256_update(&ctx, (BYTE *) &object_ids[0], - sizeof(object_ids[0]) * num_object_ids); - } - - void NextValueArgument(uint8_t *value, int64_t length) { - auto arg = fbb.CreateString((const char *) value, length); - auto empty_ids = fbb.CreateVectorOfStrings({}); - args.push_back(CreateArg(fbb, empty_ids, arg)); - sha256_update(&ctx, (BYTE *) value, length); - } - - void SetRequiredResource(const std::string &resource_name, double value) { - RAY_CHECK(resource_map_.count(resource_name) == 0); - resource_map_[resource_name] = value; - } - - uint8_t *Finish(int64_t *size) { - /* Add arguments. */ - auto arguments = fbb.CreateVector(args); - /* Update hash. */ - BYTE buff[DIGEST_SIZE]; - sha256_final(&ctx, buff); - TaskID task_id; - RAY_CHECK(sizeof(task_id) <= DIGEST_SIZE); - memcpy(&task_id, buff, sizeof(task_id)); - /* Add return object IDs. */ - std::vector> returns; - for (int64_t i = 0; i < num_returns_; i++) { - ObjectID return_id = task_compute_return_id(task_id, i); - returns.push_back(to_flatbuf(fbb, return_id)); - } - /* Create TaskInfo. */ - auto message = CreateTaskInfo( - fbb, to_flatbuf(fbb, driver_id_), to_flatbuf(fbb, task_id), - to_flatbuf(fbb, parent_task_id_), parent_counter_, - to_flatbuf(fbb, actor_creation_id_), - to_flatbuf(fbb, actor_creation_dummy_object_id_), - to_flatbuf(fbb, actor_id_), to_flatbuf(fbb, actor_handle_id_), - actor_counter_, is_actor_checkpoint_method_, - to_flatbuf(fbb, function_id_), arguments, fbb.CreateVector(returns), - map_to_flatbuf(fbb, resource_map_)); - /* Finish the TaskInfo. */ - fbb.Finish(message); - *size = fbb.GetSize(); - uint8_t *result = (uint8_t *) malloc(*size); - memcpy(result, fbb.GetBufferPointer(), *size); - fbb.Clear(); - args.clear(); - resource_map_.clear(); - return result; - } - - private: - flatbuffers::FlatBufferBuilder fbb; - std::vector> args; - SHA256_CTX ctx; - - /* Data for the builder. */ - UniqueID driver_id_; - TaskID parent_task_id_; - int64_t parent_counter_; - ActorID actor_creation_id_; - ObjectID actor_creation_dummy_object_id_; - ActorID actor_id_; - ActorID actor_handle_id_; - int64_t actor_counter_; - bool is_actor_checkpoint_method_; - FunctionID function_id_; - int64_t num_returns_; - std::unordered_map resource_map_; -}; - -TaskBuilder *make_task_builder(void) { - return new TaskBuilder(); -} - -void free_task_builder(TaskBuilder *builder) { - delete builder; -} - -bool TaskID_equal(TaskID first_id, TaskID second_id) { - return first_id == second_id; -} - -bool TaskID_is_nil(TaskID id) { - return id.is_nil(); -} - -bool ActorID_equal(ActorID first_id, ActorID second_id) { - return first_id == second_id; -} - -bool FunctionID_equal(FunctionID first_id, FunctionID second_id) { - return first_id == second_id; -} - -bool FunctionID_is_nil(FunctionID id) { - return id.is_nil(); -} - -/* Functions for building tasks. */ - -void TaskSpec_start_construct(TaskBuilder *builder, - UniqueID driver_id, - TaskID parent_task_id, - int64_t parent_counter, - ActorID actor_creation_id, - ObjectID actor_creation_dummy_object_id, - ActorID actor_id, - ActorID actor_handle_id, - int64_t actor_counter, - bool is_actor_checkpoint_method, - FunctionID function_id, - int64_t num_returns) { - builder->Start(driver_id, parent_task_id, parent_counter, actor_creation_id, - actor_creation_dummy_object_id, actor_id, actor_handle_id, - actor_counter, is_actor_checkpoint_method, function_id, - num_returns); -} - -TaskSpec *TaskSpec_finish_construct(TaskBuilder *builder, int64_t *size) { - return reinterpret_cast(builder->Finish(size)); -} - -void TaskSpec_args_add_ref(TaskBuilder *builder, - ObjectID object_ids[], - int num_object_ids) { - builder->NextReferenceArgument(&object_ids[0], num_object_ids); -} - -void TaskSpec_args_add_val(TaskBuilder *builder, - uint8_t *value, - int64_t length) { - builder->NextValueArgument(value, length); -} - -void TaskSpec_set_required_resource(TaskBuilder *builder, - const std::string &resource_name, - double value) { - builder->SetRequiredResource(resource_name, value); -} - -/* Functions for reading tasks. */ - -TaskID TaskSpec_task_id(const TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->task_id()); -} - -FunctionID TaskSpec_function(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->function_id()); -} - -ActorID TaskSpec_actor_id(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->actor_id()); -} - -ActorID TaskSpec_actor_handle_id(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->actor_handle_id()); -} - -bool TaskSpec_is_actor_task(TaskSpec *spec) { - return !TaskSpec_actor_id(spec).is_nil(); -} - -ActorID TaskSpec_actor_creation_id(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->actor_creation_id()); -} - -ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec) { - RAY_CHECK(spec); - // The task must be an actor method. - RAY_CHECK(TaskSpec_is_actor_task(spec)); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->actor_creation_dummy_object_id()); -} - -bool TaskSpec_is_actor_creation_task(TaskSpec *spec) { - return !TaskSpec_actor_creation_id(spec).is_nil(); -} - -int64_t TaskSpec_actor_counter(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return std::abs(message->actor_counter()); -} - -bool TaskSpec_is_actor_checkpoint_method(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->is_actor_checkpoint_method(); -} - -ObjectID TaskSpec_actor_dummy_object(TaskSpec *spec) { - RAY_CHECK(TaskSpec_is_actor_task(spec)); - /* The last return value for actor tasks is the dummy object that - * represents that this task has completed execution. */ - int64_t num_returns = TaskSpec_num_returns(spec); - return TaskSpec_return(spec, num_returns - 1); -} - -UniqueID TaskSpec_driver_id(const TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->driver_id()); -} - -TaskID TaskSpec_parent_task_id(const TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->parent_task_id()); -} - -int64_t TaskSpec_parent_counter(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->parent_counter(); -} - -int64_t TaskSpec_num_args(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->args()->size(); -} - -int64_t TaskSpec_num_args_by_ref(TaskSpec *spec) { - int64_t num_args = TaskSpec_num_args(spec); - int64_t num_args_by_ref = 0; - for (int64_t i = 0; i < num_args; i++) { - if (TaskSpec_arg_by_ref(spec, i)) { - num_args_by_ref++; - } - } - return num_args_by_ref; -} - -int TaskSpec_arg_id_count(TaskSpec *spec, int64_t arg_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - auto ids = message->args()->Get(arg_index)->object_ids(); - if (ids == nullptr) { - return 0; - } else { - return ids->size(); - } -} - -ObjectID TaskSpec_arg_id(TaskSpec *spec, int64_t arg_index, int64_t id_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf( - *message->args()->Get(arg_index)->object_ids()->Get(id_index)); -} - -const uint8_t *TaskSpec_arg_val(TaskSpec *spec, int64_t arg_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return (uint8_t *) message->args()->Get(arg_index)->data()->c_str(); -} - -int64_t TaskSpec_arg_length(TaskSpec *spec, int64_t arg_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->args()->Get(arg_index)->data()->size(); -} - -int64_t TaskSpec_num_returns(TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->returns()->size(); -} - -bool TaskSpec_arg_by_ref(TaskSpec *spec, int64_t arg_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return message->args()->Get(arg_index)->object_ids()->size() != 0; -} - -ObjectID TaskSpec_return(TaskSpec *spec, int64_t return_index) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return from_flatbuf(*message->returns()->Get(return_index)); -} - -double TaskSpec_get_required_resource(const TaskSpec *spec, - const std::string &resource_name) { - // This is a bit ugly. However it shouldn't be much of a performance issue - // because there shouldn't be many distinct resources in a single task spec. - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - for (size_t i = 0; i < message->required_resources()->size(); i++) { - const ResourcePair *resource_pair = message->required_resources()->Get(i); - if (string_from_flatbuf(*resource_pair->key()) == resource_name) { - return resource_pair->value(); - } - } - return 0; -} - -const std::unordered_map TaskSpec_get_required_resources( - const TaskSpec *spec) { - RAY_CHECK(spec); - auto message = flatbuffers::GetRoot(spec); - return map_from_flatbuf(*message->required_resources()); -} - -TaskSpec *TaskSpec_copy(TaskSpec *spec, int64_t task_spec_size) { - TaskSpec *copy = (TaskSpec *) malloc(task_spec_size); - memcpy(copy, spec, task_spec_size); - return copy; -} - -void TaskSpec_free(TaskSpec *spec) { - free(spec); -} - -TaskExecutionSpec::TaskExecutionSpec( - const std::vector &execution_dependencies, - const TaskSpec *spec, - int64_t task_spec_size, - int spillback_count) - : execution_dependencies_(execution_dependencies), - task_spec_size_(task_spec_size), - last_timestamp_(0), - spillback_count_(spillback_count) { - TaskSpec *spec_copy = new TaskSpec[task_spec_size_]; - memcpy(spec_copy, spec, task_spec_size); - spec_ = std::unique_ptr(spec_copy); -} - -TaskExecutionSpec::TaskExecutionSpec( - const std::vector &execution_dependencies, - const TaskSpec *spec, - int64_t task_spec_size) - : TaskExecutionSpec(execution_dependencies, spec, task_spec_size, 0) {} - -TaskExecutionSpec::TaskExecutionSpec(TaskExecutionSpec *other) - : execution_dependencies_(other->execution_dependencies_), - task_spec_size_(other->task_spec_size_), - last_timestamp_(other->last_timestamp_), - spillback_count_(other->spillback_count_) { - TaskSpec *spec_copy = new TaskSpec[task_spec_size_]; - memcpy(spec_copy, other->spec_.get(), task_spec_size_); - spec_ = std::unique_ptr(spec_copy); -} - -const std::vector &TaskExecutionSpec::ExecutionDependencies() const { - return execution_dependencies_; -} - -void TaskExecutionSpec::SetExecutionDependencies( - const std::vector &dependencies) { - execution_dependencies_ = dependencies; -} - -int64_t TaskExecutionSpec::SpecSize() const { - return task_spec_size_; -} - -int TaskExecutionSpec::SpillbackCount() const { - return spillback_count_; -} - -void TaskExecutionSpec::IncrementSpillbackCount() { - ++spillback_count_; -} - -int64_t TaskExecutionSpec::LastTimeStamp() const { - return last_timestamp_; -} - -void TaskExecutionSpec::SetLastTimeStamp(int64_t new_timestamp) { - last_timestamp_ = new_timestamp; -} - -TaskSpec *TaskExecutionSpec::Spec() const { - return spec_.get(); -} - -int64_t TaskExecutionSpec::NumDependencies() const { - TaskSpec *spec = Spec(); - int64_t num_dependencies = TaskSpec_num_args(spec); - num_dependencies += execution_dependencies_.size(); - return num_dependencies; -} - -int TaskExecutionSpec::DependencyIdCount(int64_t dependency_index) const { - TaskSpec *spec = Spec(); - /* The first dependencies are the arguments of the task itself, followed by - * the execution dependencies. Find the total number of task arguments so - * that we can index into the correct list. */ - int64_t num_args = TaskSpec_num_args(spec); - if (dependency_index < num_args) { - /* Index into the task arguments. */ - return TaskSpec_arg_id_count(spec, dependency_index); - } else { - /* Index into the execution dependencies. */ - dependency_index -= num_args; - RAY_CHECK((size_t) dependency_index < execution_dependencies_.size()); - /* All elements in the execution dependency list have exactly one ID. */ - return 1; - } -} - -ObjectID TaskExecutionSpec::DependencyId(int64_t dependency_index, - int64_t id_index) const { - TaskSpec *spec = Spec(); - /* The first dependencies are the arguments of the task itself, followed by - * the execution dependencies. Find the total number of task arguments so - * that we can index into the correct list. */ - int64_t num_args = TaskSpec_num_args(spec); - if (dependency_index < num_args) { - /* Index into the task arguments. */ - return TaskSpec_arg_id(spec, dependency_index, id_index); - } else { - /* Index into the execution dependencies. */ - dependency_index -= num_args; - RAY_CHECK((size_t) dependency_index < execution_dependencies_.size()); - return execution_dependencies_[dependency_index]; - } -} - -bool TaskExecutionSpec::DependsOn(ObjectID object_id) const { - // Iterate through the task arguments to see if it contains object_id. - TaskSpec *spec = Spec(); - int64_t num_args = TaskSpec_num_args(spec); - for (int i = 0; i < num_args; ++i) { - int count = TaskSpec_arg_id_count(spec, i); - for (int j = 0; j < count; j++) { - ObjectID arg_id = TaskSpec_arg_id(spec, i, j); - if (arg_id == object_id) { - return true; - } - } - } - // Iterate through the execution dependencies to see if it contains object_id. - for (auto dependency_id : execution_dependencies_) { - if (dependency_id == object_id) { - return true; - } - } - // The requested object ID was not a task argument or an execution dependency. - // This task is not dependent on it. - return false; -} - -bool TaskExecutionSpec::IsStaticDependency(int64_t dependency_index) const { - TaskSpec *spec = Spec(); - /* The first dependencies are the arguments of the task itself, followed by - * the execution dependencies. If the requested dependency index is a task - * argument, then it is a task dependency. */ - int64_t num_args = TaskSpec_num_args(spec); - return (dependency_index < num_args); -} - -/* TASK INSTANCES */ - -Task *Task_alloc(const TaskSpec *spec, - int64_t task_spec_size, - TaskStatus state, - DBClientID local_scheduler_id, - const std::vector &execution_dependencies) { - Task *result = new Task(); - auto execution_spec = - new TaskExecutionSpec(execution_dependencies, spec, task_spec_size); - result->execution_spec = std::unique_ptr(execution_spec); - result->state = state; - result->local_scheduler_id = local_scheduler_id; - return result; -} - -Task *Task_alloc(TaskExecutionSpec &execution_spec, - TaskStatus state, - DBClientID local_scheduler_id) { - Task *result = new Task(); - result->execution_spec = std::unique_ptr( - new TaskExecutionSpec(&execution_spec)); - result->state = state; - result->local_scheduler_id = local_scheduler_id; - return result; -} - -Task *Task_copy(Task *other) { - return Task_alloc(*Task_task_execution_spec(other), other->state, - other->local_scheduler_id); -} - -int64_t Task_size(Task *task_arg) { - return sizeof(Task) - sizeof(TaskSpec) + task_arg->execution_spec->SpecSize(); -} - -TaskStatus Task_state(Task *task) { - return task->state; -} - -void Task_set_state(Task *task, TaskStatus state) { - task->state = state; -} - -DBClientID Task_local_scheduler(Task *task) { - return task->local_scheduler_id; -} - -void Task_set_local_scheduler(Task *task, DBClientID local_scheduler_id) { - task->local_scheduler_id = local_scheduler_id; -} - -TaskExecutionSpec *Task_task_execution_spec(Task *task) { - return task->execution_spec.get(); -} - -TaskID Task_task_id(Task *task) { - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - TaskSpec *spec = execution_spec->Spec(); - return TaskSpec_task_id(spec); -} - -void Task_free(Task *task) { - delete task; -} diff --git a/src/common/task.h b/src/common/task.h deleted file mode 100644 index 3984cfdd5..000000000 --- a/src/common/task.h +++ /dev/null @@ -1,609 +0,0 @@ -#ifndef TASK_H -#define TASK_H - -#include - -#include -#include -#include "common.h" - -#include - -#include "format/common_generated.h" - -using namespace ray; - -typedef char TaskSpec; - -class TaskExecutionSpec { - public: - TaskExecutionSpec(const std::vector &execution_dependencies, - const TaskSpec *spec, - int64_t task_spec_size); - TaskExecutionSpec(const std::vector &execution_dependencies, - const TaskSpec *spec, - int64_t task_spec_size, - int spillback_count); - TaskExecutionSpec(TaskExecutionSpec *execution_spec); - - /// Get the task's execution dependencies. - /// - /// @return A vector of object IDs representing this task's execution - /// dependencies. - const std::vector &ExecutionDependencies() const; - - /// Set the task's execution dependencies. - /// - /// @param dependencies The value to set the execution dependencies to. - /// @return Void. - void SetExecutionDependencies(const std::vector &dependencies); - - /// Get the task spec size. - /// - /// @return The size of the immutable task spec. - int64_t SpecSize() const; - - /// Get the task's spillback count, which tracks the number of times - /// this task was spilled back from local to the global scheduler. - /// - /// @return The spillback count for this task. - int SpillbackCount() const; - - /// Increment the spillback count for this task. - /// - /// @return Void. - void IncrementSpillbackCount(); - - /// Get the task's last timestamp. - /// - /// @return The timestamp when this task was last received for scheduling. - int64_t LastTimeStamp() const; - - /// Set the task's last timestamp to the specified value. - /// - /// @param new_timestamp The new timestamp in millisecond to set the task's - /// time stamp to. Tracks the last time this task entered a local - /// scheduler. - /// @return Void. - void SetLastTimeStamp(int64_t new_timestamp); - - /// Get the task spec. - /// - /// @return A pointer to the immutable task spec. - TaskSpec *Spec() const; - - /// Get the number of dependencies. This comprises the immutable task - /// arguments and the mutable execution dependencies. - /// - /// @return The number of dependencies. - int64_t NumDependencies() const; - - /// Get the number of object IDs at the given dependency index. - /// - /// @param dependency_index The dependency index whose object IDs to count. - /// @return The number of object IDs at the given dependency_index. - int DependencyIdCount(int64_t dependency_index) const; - - /// Get the object ID of a given dependency index. - /// - /// @param dependency_index The index at which we should look up the object - /// ID. - /// @param id_index The index of the object ID. - ObjectID DependencyId(int64_t dependency_index, int64_t id_index) const; - - /// Compute whether the task is dependent on an object ID. - /// - /// @param object_id The object ID that the task may be dependent on. - /// @return bool This returns true if the task is dependent on the given - /// object ID and false otherwise. - bool DependsOn(ObjectID object_id) const; - - /// Returns whether the given dependency index is a static dependency (an - /// argument of the immutable task). - /// - /// @param dependency_index The requested dependency index. - /// @return bool This returns true if the requested dependency index is - /// immutable (an argument of the task). - bool IsStaticDependency(int64_t dependency_index) const; - - private: - /** A list of object IDs representing this task's dependencies at execution - * time. */ - std::vector execution_dependencies_; - /** The size of the task specification for this task. */ - int64_t task_spec_size_; - /** Last time this task was received for scheduling. */ - int64_t last_timestamp_; - /** Number of times this task was spilled back by local schedulers. */ - int spillback_count_; - /** The task specification for this task. */ - std::unique_ptr spec_; -}; - -class TaskBuilder; - -typedef UniqueID FunctionID; - -/** The task ID is a deterministic hash of the function ID that the task - * executes and the argument IDs or argument values. */ -typedef UniqueID TaskID; - -/** The actor ID is the ID of the actor that a task must run on. If the task is - * not run on an actor, then NIL_ACTOR_ID should be used. */ -typedef UniqueID ActorID; - -/** - * Compare two task IDs. - * - * @param first_id The first task ID to compare. - * @param second_id The first task ID to compare. - * @return True if the task IDs are the same and false otherwise. - */ -bool TaskID_equal(TaskID first_id, TaskID second_id); - -/** - * Compare a task ID to the nil ID. - * - * @param id The task ID to compare to nil. - * @return True if the task ID is equal to nil. - */ -bool TaskID_is_nil(TaskID id); - -/** - * Compare two actor IDs. - * - * @param first_id The first actor ID to compare. - * @param second_id The first actor ID to compare. - * @return True if the actor IDs are the same and false otherwise. - */ -bool ActorID_equal(ActorID first_id, ActorID second_id); - -/** - * Compare two function IDs. - * - * @param first_id The first function ID to compare. - * @param second_id The first function ID to compare. - * @return True if the function IDs are the same and false otherwise. - */ -bool FunctionID_equal(FunctionID first_id, FunctionID second_id); - -/** - * Compare a function ID to the nil ID. - * - * @param id The function ID to compare to nil. - * @return True if the function ID is equal to nil. - */ -bool FunctionID_is_nil(FunctionID id); - -/* Construct and modify task specifications. */ - -TaskBuilder *make_task_builder(void); - -void free_task_builder(TaskBuilder *builder); - -/** - * Begin constructing a task_spec. After this is called, the arguments must be - * added to the task_spec and then finish_construct_task_spec must be called. - * - * @param driver_id The ID of the driver whose job is responsible for the - * creation of this task. - * @param parent_task_id The task ID of the task that submitted this task. - * @param parent_counter A counter indicating how many tasks were submitted by - * the parent task prior to this one. - * @param actor_creation_id The actor creation ID of this task. - * @param actor_creation_dummy_object_id The dummy object for the corresponding - * actor creation task, assuming this is an actor method. - * @param actor_id The ID of the actor that this task is for. If it is not an - * actor task, then this if NIL_ACTOR_ID. - * @param actor_handle_id The ID of the actor handle that this task was - * submitted through. If it is not an actor task, or if this is the - * original handle, then this is NIL_ACTOR_ID. - * @param actor_counter A counter indicating how many tasks have been submitted - * to the same actor before this one. - * @param is_actor_checkpoint_method True if this is an actor checkpoint method - * and false otherwise. - * @param function_id The function ID of the function to execute in this task. - * @param num_args The number of arguments that this task has. - * @param num_returns The number of return values that this task has. - * @param args_value_size The total size in bytes of the arguments to this task - ignoring object ID arguments. - * @return The partially constructed task_spec. - */ -void TaskSpec_start_construct(TaskBuilder *B, - UniqueID driver_id, - TaskID parent_task_id, - int64_t parent_counter, - ActorID actor_creation_id, - ObjectID actor_creation_dummy_object_id, - ActorID actor_id, - ActorHandleID actor_handle_id, - int64_t actor_counter, - bool is_actor_checkpoint_method, - FunctionID function_id, - int64_t num_returns); - -/** - * Finish constructing a task_spec. This computes the task ID and the object IDs - * of the task return values. This must be called after all of the arguments - * have been added to the task. - * - * @param spec The task spec whose ID and return object IDs should be computed. - * @return Void. - */ -TaskSpec *TaskSpec_finish_construct(TaskBuilder *builder, int64_t *size); - -/** - * Return the function ID of the task. - * - * @param spec The task_spec in question. - * @return The function ID of the function to execute in this task. - */ -FunctionID TaskSpec_function(TaskSpec *spec); - -/** - * Return the actor ID of the task. - * - * @param spec The task_spec in question. - * @return The actor ID of the actor the task is part of. - */ -ActorID TaskSpec_actor_id(TaskSpec *spec); - -/** - * Return the actor handle ID of the task. - * - * @param spec The task_spec in question. - * @return The ID of the actor handle that the task was submitted through. - */ -ActorID TaskSpec_actor_handle_id(TaskSpec *spec); - -/** - * Return whether this task is for an actor. - * - * @param spec The task_spec in question. - * @return Whether the task is for an actor. - */ -bool TaskSpec_is_actor_task(TaskSpec *spec); - -/// Return whether this task is an actor creation task or not. -/// -/// \param spec The task_spec in question. -/// \return True if this task is an actor creation task and false otherwise. -bool TaskSpec_is_actor_creation_task(TaskSpec *spec); - -/// Return the actor creation ID of the task. The task must be an actor creation -/// task. -/// -/// \param spec The task_spec in question. -/// \return The actor creation ID if this is an actor creation task. -ActorID TaskSpec_actor_creation_id(TaskSpec *spec); - -/// Return the actor creation dummy object ID of the task. The task must be an -/// actor task. -/// -/// \param spec The task_spec in question. -/// \return The actor creation dummy object ID corresponding to this actor task. -ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec); - -/** - * Return the actor counter of the task. This starts at 0 and increments by 1 - * every time a new task is submitted to run on the actor. - * - * @param spec The task_spec in question. - * @return The actor counter of the task. - */ -int64_t TaskSpec_actor_counter(TaskSpec *spec); - -/** - * Return whether the task is a checkpoint method execution. - * - * @param spec The task_spec in question. - * @return Whether the task is a checkpoint method. - */ -bool TaskSpec_is_actor_checkpoint_method(TaskSpec *spec); - -/** - * Return an actor task's dummy return value. Dummy objects are used to - * encode an actor's state dependencies in the task graph. The dummy object - * is local if and only if the task that returned it has completed - * execution. - * - * @param spec The task_spec in question. - * @return The dummy object ID that the actor task will return. - */ -ObjectID TaskSpec_actor_dummy_object(TaskSpec *spec); - -/** - * Return the driver ID of the task. - * - * @param spec The task_spec in question. - * @return The driver ID of the task. - */ -UniqueID TaskSpec_driver_id(const TaskSpec *spec); - -/** - * Return the task ID of the parent task. - * - * @param spec The task_spec in question. - * @return The task ID of the parent task. - */ -TaskID TaskSpec_parent_task_id(const TaskSpec *spec); - -/** - * Return the task counter of the parent task. For example, this equals 5 if - * this task was the 6th task submitted by the parent task. - * - * @param spec The task_spec in question. - * @return The task counter of the parent task. - */ -int64_t TaskSpec_parent_counter(TaskSpec *spec); - -/** - * Return the task ID of the task. - * - * @param spec The task_spec in question. - * @return The task ID of the task. - */ -TaskID TaskSpec_task_id(const TaskSpec *spec); - -/** - * Get the number of arguments to this task. - * - * @param spec The task_spec in question. - * @return The number of arguments to this task. - */ -int64_t TaskSpec_num_args(TaskSpec *spec); - -/** - * Get the number of return values expected from this task. - * - * @param spec The task_spec in question. - * @return The number of return values expected from this task. - */ -int64_t TaskSpec_num_returns(TaskSpec *spec); - -/** - * Return true if this argument is passed by reference. - * - * @param spec The task_spec in question. - * @param arg_index The index of the argument in question. - * @return True if this argument is passed by reference. - */ -bool TaskSpec_arg_by_ref(TaskSpec *spec, int64_t arg_index); - -/** - * Get number of object IDs in a given argument - * - * @param spec The task_spec in question. - * @param arg_index The index of the argument in question. - * @return number of object IDs in this argument - */ -int TaskSpec_arg_id_count(TaskSpec *spec, int64_t arg_index); - -/** - * Get a particular argument to this task. This assumes the argument is an - * object ID. - * - * @param spec The task_spec in question. - * @param arg_index The index of the argument in question. - * @param id_index The index of the object ID in this arg. - * @return The argument at that index. - */ -ObjectID TaskSpec_arg_id(TaskSpec *spec, int64_t arg_index, int64_t id_index); - -/** - * Get a particular argument to this task. This assumes the argument is a value. - * - * @param spec The task_spec in question. - * @param arg_index The index of the argument in question. - * @return The argument at that index. - */ -const uint8_t *TaskSpec_arg_val(TaskSpec *spec, int64_t arg_index); - -/** - * Get the number of bytes in a particular argument to this task. This assumes - * the argument is a value. - * - * @param spec The task_spec in question. - * @param arg_index The index of the argument in question. - * @return The number of bytes in the argument. - */ -int64_t TaskSpec_arg_length(TaskSpec *spec, int64_t arg_index); - -/** - * Set the next task argument. Note that this API only allows you to set the - * arguments in their order of appearance. - * - * @param spec The task_spec in question. - * @param object_ids The object IDs to set the argument to. - * @param num_object_ids number of IDs in this param, usually 1. - * @return The number of task arguments that have been set before this one. This - * is only used for testing. - */ -void TaskSpec_args_add_ref(TaskBuilder *spec, - ObjectID object_ids[], - int num_object_ids); - -/** - * Set the next task argument. Note that this API only allows you to set the - * arguments in their order of appearance. - * - * @param spec The task_spec in question. - * @param The value to set the argument to. - * @param The length of the value to set the argument to. - * @return The number of task arguments that have been set before this one. This - * is only used for testing. - */ -void TaskSpec_args_add_val(TaskBuilder *builder, - uint8_t *value, - int64_t length); - -/** - * Set the value associated to a resource index. - * - * @param spec Task specification. - * @param resource_name Name of the resource in the resource vector. - * @param value Value for the resource. This can be a quantity of this resource - * this task needs or a value for an attribute this task requires. - * @return Void. - */ -void TaskSpec_set_required_resource(TaskBuilder *builder, - const std::string &resource_name, - double value); - -/** - * Get a particular return object ID of a task. - * - * @param spec The task_spec in question. - * @param return_index The index of the return object ID in question. - * @return The relevant return object ID. - */ -ObjectID TaskSpec_return(TaskSpec *data, int64_t return_index); - -/** - * Get the value associated to a resource name. - * - * @param spec Task specification. - * @param resource_name Name of the resource. - * @return How many of this resource the task needs to execute. - */ -double TaskSpec_get_required_resource(const TaskSpec *spec, - const std::string &resource_name); - -/** - * - */ -const std::unordered_map TaskSpec_get_required_resources( - const TaskSpec *spec); - -/** - * Compute the object id associated to a put call. - * - * @param task_id The task id of the parent task that called the put. - * @param put_index The number of put calls in this task so far. - * @return The object ID for the object that was put. - */ -ObjectID task_compute_put_id(TaskID task_id, int64_t put_index); - -/** - * Print the task as a humanly readable string. - * - * @param spec The task_spec in question. - * @return The humanly readable string. - */ -std::string TaskSpec_print(TaskSpec *spec); - -/** - * Create a copy of the task spec. Must be freed with TaskSpec_free after use. - * - * @param spec The task specification that will be copied. - * @param task_spec_size The size of the task specification in bytes. - * @returns Pointer to the copy of the task specification. - */ -TaskSpec *TaskSpec_copy(TaskSpec *spec, int64_t task_spec_size); - -/** - * Free a task_spec. - * - * @param The task_spec in question. - * @return Void. - */ -void TaskSpec_free(TaskSpec *spec); - -/** - * ==== Task ==== - * Contains information about a scheduled task: The task specification, the - * task scheduling state (WAITING, SCHEDULED, QUEUED, RUNNING, DONE), and which - * local scheduler the task is scheduled on. - */ - -/** The scheduling_state can be used as a flag when we are listening - * for an event, for example TASK_WAITING | TASK_SCHEDULED. */ -enum class TaskStatus : uint { - /** The task is waiting to be scheduled. */ - WAITING = 1, - /** The task has been scheduled to a node, but has not been queued yet. */ - SCHEDULED = 2, - /** The task has been queued on a node, where it will wait for its - * dependencies to become ready and a worker to become available. */ - QUEUED = 4, - /** The task is running on a worker. */ - RUNNING = 8, - /** The task is done executing. */ - DONE = 16, - /** The task was not able to finish. */ - LOST = 32, - /** The task will be submitted for reexecution. */ - RECONSTRUCTING = 64, - /** An actor task is cached at a local scheduler and is waiting for the - * corresponding actor to be created. */ - ACTOR_CACHED = 128 -}; - -inline TaskStatus operator|(const TaskStatus &a, const TaskStatus &b) { - uint c = static_cast(a) | static_cast(b); - return static_cast(c); -} - -/** A task is an execution of a task specification. It has a state of execution - * (see scheduling_state) and the ID of the local scheduler it is scheduled on - * or running on. */ - -struct Task { - /** The scheduling state of the task. */ - TaskStatus state; - /** The ID of the local scheduler involved. */ - DBClientID local_scheduler_id; - /** The execution specification for this task. */ - std::unique_ptr execution_spec; -}; - -/** - * Allocate a new task. Must be freed with free_task after use. - * - * @param spec The task spec for the new task. - * @param state The scheduling state for the new task. - * @param local_scheduler_id The ID of the local scheduler that the task is - * scheduled on, if any. - */ -Task *Task_alloc(const TaskSpec *spec, - int64_t task_spec_size, - TaskStatus state, - DBClientID local_scheduler_id, - const std::vector &execution_dependencies); - -Task *Task_alloc(TaskExecutionSpec &execution_spec, - TaskStatus state, - DBClientID local_scheduler_id); - -/** - * Create a copy of the task. Must be freed with Task_free after use. - * - * @param other The task that will be copied. - * @returns Pointer to the copy of the task. - */ -Task *Task_copy(Task *other); - -/** Size of task structure in bytes. */ -int64_t Task_size(Task *task); - -/** The scheduling state of the task. */ -TaskStatus Task_state(Task *task); - -/** Update the schedule state of the task. */ -void Task_set_state(Task *task, TaskStatus state); - -/** Local scheduler this task has been assigned to or is running on. */ -DBClientID Task_local_scheduler(Task *task); - -/** Set the local scheduler ID for this task. */ -void Task_set_local_scheduler(Task *task, DBClientID local_scheduler_id); - -TaskExecutionSpec *Task_task_execution_spec(Task *task); - -/** Task ID of this task. */ -TaskID Task_task_id(Task *task); - -/** Free this task datastructure. */ -void Task_free(Task *task); - -#endif /* TASK_H */ diff --git a/src/common/test/db_tests.cc b/src/common/test/db_tests.cc deleted file mode 100644 index 83585ca66..000000000 --- a/src/common/test/db_tests.cc +++ /dev/null @@ -1,246 +0,0 @@ -#include "greatest.h" - -#include -#include -#include - -#include "event_loop.h" -#include "test_common.h" -#include "example_task.h" -#include "net.h" -#include "state/db.h" -#include "state/db_client_table.h" -#include "state/object_table.h" -#include "state/task_table.h" -#include "state/redis.h" -#include "task.h" - -SUITE(db_tests); - -TaskBuilder *g_task_builder = NULL; - -/* Retry 10 times with an 100ms timeout. */ -const int NUM_RETRIES = 10; -const uint64_t TIMEOUT = 50; - -const char *manager_addr = "127.0.0.1"; -int manager_port1 = 12345; -int manager_port2 = 12346; -char received_addr1[16] = {0}; -int received_port1; -char received_addr2[16] = {0}; -int received_port2; - -typedef struct { int test_number; } user_context; - -const int TEST_NUMBER = 10; - -/* Test if entries have been written to the database. */ - -void lookup_done_callback(ObjectID object_id, - bool never_created, - const std::vector &manager_ids, - void *user_context) { - DBHandle *db = (DBHandle *) user_context; - RAY_CHECK(manager_ids.size() == 2); - const std::vector managers = - db_client_table_get_ip_addresses(db, manager_ids); - RAY_CHECK(parse_ip_addr_port(managers.at(0).c_str(), received_addr1, - &received_port1) == 0); - RAY_CHECK(parse_ip_addr_port(managers.at(1).c_str(), received_addr2, - &received_port2) == 0); -} - -/* Entry added to database successfully. */ -void add_done_callback(ObjectID object_id, bool success, void *user_context) {} - -/* Test if we got a timeout callback if we couldn't connect database. */ -void timeout_callback(ObjectID object_id, void *context, void *user_data) { - user_context *uc = (user_context *) context; - RAY_CHECK(uc->test_number == TEST_NUMBER); -} - -int64_t timeout_handler(event_loop *loop, int64_t id, void *context) { - event_loop_stop(loop); - return EVENT_LOOP_TIMER_DONE; -} - -TEST object_table_lookup_test(void) { - event_loop *loop = event_loop_create(); - /* This uses manager_port1. */ - std::vector db_connect_args1; - db_connect_args1.push_back("manager_address"); - db_connect_args1.push_back("127.0.0.1:12345"); - DBHandle *db1 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - manager_addr, db_connect_args1); - /* This uses manager_port2. */ - std::vector db_connect_args2; - db_connect_args2.push_back("manager_address"); - db_connect_args2.push_back("127.0.0.1:12346"); - DBHandle *db2 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - manager_addr, db_connect_args2); - db_attach(db1, loop, false); - db_attach(db2, loop, false); - UniqueID id = UniqueID::from_random(); - RetryInfo retry = { - .num_retries = NUM_RETRIES, - .timeout = TIMEOUT, - .fail_callback = timeout_callback, - }; - object_table_add(db1, id, 0, (unsigned char *) NIL_DIGEST, &retry, - add_done_callback, NULL); - object_table_add(db2, id, 0, (unsigned char *) NIL_DIGEST, &retry, - add_done_callback, NULL); - event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler, - NULL); - event_loop_run(loop); - object_table_lookup(db1, id, &retry, lookup_done_callback, db1); - event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler, - NULL); - event_loop_run(loop); - ASSERT_STR_EQ(&received_addr1[0], manager_addr); - ASSERT((received_port1 == manager_port1 && received_port2 == manager_port2) || - (received_port2 == manager_port1 && received_port1 == manager_port2)); - - db_disconnect(db1); - db_disconnect(db2); - - destroy_outstanding_callbacks(loop); - event_loop_destroy(loop); - PASS(); -} - -int task_table_test_callback_called = 0; -Task *task_table_test_task; - -void task_table_test_fail_callback(UniqueID id, - void *context, - void *user_data) { - event_loop *loop = (event_loop *) user_data; - event_loop_stop(loop); -} - -int64_t task_table_delayed_add_task(event_loop *loop, - int64_t id, - void *context) { - DBHandle *db = (DBHandle *) context; - RetryInfo retry = { - .num_retries = NUM_RETRIES, - .timeout = TIMEOUT, - .fail_callback = task_table_test_fail_callback, - }; - task_table_add_task(db, Task_copy(task_table_test_task), &retry, NULL, - (void *) loop); - return EVENT_LOOP_TIMER_DONE; -} - -void task_table_test_callback(Task *callback_task, void *user_data) { - task_table_test_callback_called = 1; - RAY_CHECK(Task_state(callback_task) == TaskStatus::SCHEDULED); - RAY_CHECK(Task_size(callback_task) == Task_size(task_table_test_task)); - RAY_CHECK(Task_equals(callback_task, task_table_test_task)); - event_loop *loop = (event_loop *) user_data; - event_loop_stop(loop); -} - -TEST task_table_test(void) { - task_table_test_callback_called = 0; - event_loop *loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler", - "127.0.0.1", std::vector()); - db_attach(db, loop, false); - DBClientID local_scheduler_id = DBClientID::from_random(); - TaskExecutionSpec spec = example_task_execution_spec(1, 1); - task_table_test_task = - Task_alloc(spec, TaskStatus::SCHEDULED, local_scheduler_id); - RetryInfo retry = { - .num_retries = NUM_RETRIES, - .timeout = TIMEOUT, - .fail_callback = task_table_test_fail_callback, - }; - task_table_subscribe(db, local_scheduler_id, TaskStatus::SCHEDULED, - task_table_test_callback, (void *) loop, &retry, NULL, - (void *) loop); - event_loop_add_timer( - loop, 200, (event_loop_timer_handler) task_table_delayed_add_task, db); - event_loop_run(loop); - Task_free(task_table_test_task); - db_disconnect(db); - destroy_outstanding_callbacks(loop); - event_loop_destroy(loop); - ASSERT(task_table_test_callback_called); - PASS(); -} - -int num_test_callback_called = 0; - -void task_table_all_test_callback(Task *task, void *user_data) { - num_test_callback_called += 1; -} - -TEST task_table_all_test(void) { - event_loop *loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler", - "127.0.0.1", std::vector()); - db_attach(db, loop, false); - TaskExecutionSpec spec = example_task_execution_spec(1, 1); - /* Schedule two tasks on different local local schedulers. */ - Task *task1 = - Task_alloc(spec, TaskStatus::SCHEDULED, DBClientID::from_random()); - Task *task2 = - Task_alloc(spec, TaskStatus::SCHEDULED, DBClientID::from_random()); - RetryInfo retry = { - .num_retries = NUM_RETRIES, .timeout = TIMEOUT, .fail_callback = NULL, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::SCHEDULED, - task_table_all_test_callback, NULL, &retry, NULL, NULL); - event_loop_add_timer(loop, 50, (event_loop_timer_handler) timeout_handler, - NULL); - event_loop_run(loop); - /* TODO(pcm): Get rid of this sleep once the robust pubsub is implemented. */ - task_table_add_task(db, task1, &retry, NULL, NULL); - task_table_add_task(db, task2, &retry, NULL, NULL); - event_loop_add_timer(loop, 200, (event_loop_timer_handler) timeout_handler, - NULL); - event_loop_run(loop); - db_disconnect(db); - destroy_outstanding_callbacks(loop); - event_loop_destroy(loop); - ASSERT(num_test_callback_called == 2); - PASS(); -} - -TEST unique_client_id_test(void) { - const int num_conns = 100; - - DBClientID ids[num_conns]; - DBHandle *db; - for (int i = 0; i < num_conns; ++i) { - db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - ids[i] = get_db_client_id(db); - db_disconnect(db); - } - for (int i = 0; i < num_conns; ++i) { - for (int j = 0; j < i; ++j) { - ASSERT(!(ids[i] == ids[j])); - } - } - PASS(); -} - -SUITE(db_tests) { - RUN_REDIS_TEST(object_table_lookup_test); - RUN_REDIS_TEST(task_table_test); - RUN_REDIS_TEST(task_table_all_test); - RUN_REDIS_TEST(unique_client_id_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - g_task_builder = make_task_builder(); - GREATEST_MAIN_BEGIN(); - RUN_SUITE(db_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/example_task.h b/src/common/test/example_task.h deleted file mode 100644 index f90cab68f..000000000 --- a/src/common/test/example_task.h +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef EXAMPLE_TASK_H -#define EXAMPLE_TASK_H - -#include "task.h" - -extern TaskBuilder *g_task_builder; - -const int64_t arg_value_size = 1000; - -static inline TaskExecutionSpec example_task_execution_spec_with_args( - int64_t num_args, - int64_t num_returns, - ObjectID arg_ids[]) { - TaskID parent_task_id = TaskID::from_random(); - FunctionID func_id = FunctionID::from_random(); - TaskSpec_start_construct(g_task_builder, UniqueID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, num_returns); - for (int64_t i = 0; i < num_args; ++i) { - ObjectID arg_id; - if (arg_ids == NULL) { - arg_id = ObjectID::from_random(); - } else { - arg_id = arg_ids[i]; - } - TaskSpec_args_add_ref(g_task_builder, &arg_id, 1); - } - int64_t task_spec_size; - TaskSpec *spec = TaskSpec_finish_construct(g_task_builder, &task_spec_size); - std::vector execution_dependencies; - auto execution_spec = - TaskExecutionSpec(execution_dependencies, spec, task_spec_size); - TaskSpec_free(spec); - return execution_spec; -} - -static inline TaskExecutionSpec example_task_execution_spec( - int64_t num_args, - int64_t num_returns) { - return example_task_execution_spec_with_args(num_args, num_returns, NULL); -} - -static inline Task *example_task_with_args(int64_t num_args, - int64_t num_returns, - TaskStatus task_state, - ObjectID arg_ids[]) { - TaskExecutionSpec spec = - example_task_execution_spec_with_args(num_args, num_returns, arg_ids); - Task *instance = Task_alloc(spec, task_state, UniqueID::nil()); - return instance; -} - -static inline Task *example_task(int64_t num_args, - int64_t num_returns, - TaskStatus task_state) { - TaskExecutionSpec spec = example_task_execution_spec(num_args, num_returns); - Task *instance = Task_alloc(spec, task_state, UniqueID::nil()); - return instance; -} - -static inline bool Task_equals(Task *task1, Task *task2) { - if (task1->state != task2->state) { - return false; - } - if (!(task1->local_scheduler_id == task2->local_scheduler_id)) { - return false; - } - auto execution_spec1 = Task_task_execution_spec(task1); - auto execution_spec2 = Task_task_execution_spec(task2); - if (execution_spec1->SpecSize() != execution_spec2->SpecSize()) { - return false; - } - return memcmp(execution_spec1->Spec(), execution_spec2->Spec(), - execution_spec1->SpecSize()) == 0; -} - -#endif /* EXAMPLE_TASK_H */ diff --git a/src/common/test/io_tests.cc b/src/common/test/io_tests.cc deleted file mode 100644 index 092ca97b7..000000000 --- a/src/common/test/io_tests.cc +++ /dev/null @@ -1,114 +0,0 @@ -#include "greatest.h" - -#include -#include -#include - -#include -#include - -#include "io.h" - -SUITE(io_tests); - -TEST ipc_socket_test(void) { -#ifndef _WIN32 - const char *socket_pathname = "/tmp/test-socket"; - int socket_fd = bind_ipc_sock(socket_pathname, true); - ASSERT(socket_fd >= 0); - - const char *test_string = "hello world"; - const char *test_bytes = "another string"; - pid_t pid = fork(); - if (pid == 0) { - close(socket_fd); - socket_fd = connect_ipc_sock(socket_pathname); - ASSERT(socket_fd >= 0); - write_log_message(socket_fd, test_string); - write_message(socket_fd, - static_cast(CommonMessageType::LOG_MESSAGE), - strlen(test_bytes), (uint8_t *) test_bytes); - close(socket_fd); - exit(0); - } else { - int client_fd = accept_client(socket_fd); - ASSERT(client_fd >= 0); - char *message = read_log_message(client_fd); - ASSERT(message != NULL); - ASSERT_STR_EQ(test_string, message); - free(message); - int64_t type; - int64_t len; - uint8_t *bytes; - read_message(client_fd, &type, &len, &bytes); - ASSERT(static_cast(type) == - CommonMessageType::LOG_MESSAGE); - ASSERT(memcmp(test_bytes, bytes, len) == 0); - free(bytes); - close(client_fd); - close(socket_fd); - unlink(socket_pathname); - } -#endif - PASS(); -} - -TEST long_ipc_socket_test(void) { -#ifndef _WIN32 - const char *socket_pathname = "/tmp/long-test-socket"; - int socket_fd = bind_ipc_sock(socket_pathname, true); - ASSERT(socket_fd >= 0); - - std::stringstream test_string_ss; - for (int i = 0; i < 10000; i++) { - test_string_ss << "hello world "; - } - std::string test_string = test_string_ss.str(); - const char *test_bytes = "another string"; - pid_t pid = fork(); - if (pid == 0) { - close(socket_fd); - socket_fd = connect_ipc_sock(socket_pathname); - ASSERT(socket_fd >= 0); - write_log_message(socket_fd, test_string.c_str()); - write_message(socket_fd, - static_cast(CommonMessageType::LOG_MESSAGE), - strlen(test_bytes), (uint8_t *) test_bytes); - close(socket_fd); - exit(0); - } else { - int client_fd = accept_client(socket_fd); - ASSERT(client_fd >= 0); - char *message = read_log_message(client_fd); - ASSERT(message != NULL); - ASSERT_STR_EQ(test_string.c_str(), message); - free(message); - int64_t type; - int64_t len; - uint8_t *bytes; - read_message(client_fd, &type, &len, &bytes); - ASSERT(static_cast(type) == - CommonMessageType::LOG_MESSAGE); - ASSERT(memcmp(test_bytes, bytes, len) == 0); - free(bytes); - close(client_fd); - close(socket_fd); - unlink(socket_pathname); - } - -#endif - PASS(); -} - -SUITE(io_tests) { - RUN_TEST(ipc_socket_test); - RUN_TEST(long_ipc_socket_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); - RUN_SUITE(io_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/object_table_tests.cc b/src/common/test/object_table_tests.cc deleted file mode 100644 index 059972438..000000000 --- a/src/common/test/object_table_tests.cc +++ /dev/null @@ -1,919 +0,0 @@ -#include "greatest.h" - -#include "event_loop.h" -#include "example_task.h" -#include "test_common.h" -#include "common.h" -#include "state/db_client_table.h" -#include "state/object_table.h" -#include "state/redis.h" - -#include - -SUITE(object_table_tests); - -static event_loop *g_loop; -TaskBuilder *g_task_builder = NULL; - -/* ==== Test adding and looking up metadata ==== */ - -int new_object_failed = 0; -int new_object_succeeded = 0; -ObjectID new_object_id; -Task *new_object_task; -TaskSpec *new_object_task_spec; -TaskID new_object_task_id; - -void new_object_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - new_object_failed = 1; - event_loop_stop(g_loop); -} - -/* === Test adding an object with an associated task === */ - -void new_object_done_callback(ObjectID object_id, - TaskID task_id, - bool is_put, - void *user_context) { - new_object_succeeded = 1; - RAY_CHECK(object_id == new_object_id); - RAY_CHECK(task_id == new_object_task_id); - event_loop_stop(g_loop); -} - -void new_object_lookup_callback(ObjectID object_id, void *user_context) { - RAY_CHECK(object_id == new_object_id); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = new_object_fail_callback, - }; - DBHandle *db = (DBHandle *) user_context; - result_table_lookup(db, new_object_id, &retry, new_object_done_callback, - NULL); -} - -void new_object_task_callback(TaskID task_id, void *user_context) { - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = new_object_fail_callback, - }; - DBHandle *db = (DBHandle *) user_context; - result_table_add(db, new_object_id, new_object_task_id, false, &retry, - new_object_lookup_callback, (void *) db); -} - -void task_table_subscribe_done(TaskID task_id, void *user_context) { - RetryInfo retry = { - .num_retries = 5, .timeout = 100, .fail_callback = NULL, - }; - DBHandle *db = (DBHandle *) user_context; - task_table_add_task(db, Task_copy(new_object_task), &retry, - new_object_task_callback, db); -} - -TEST new_object_test(void) { - new_object_failed = 0; - new_object_succeeded = 0; - new_object_id = ObjectID::from_random(); - new_object_task = example_task(1, 1, TaskStatus::WAITING); - new_object_task_spec = Task_task_execution_spec(new_object_task)->Spec(); - new_object_task_id = TaskSpec_task_id(new_object_task_spec); - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = new_object_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, task_table_subscribe_done, db); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(new_object_succeeded); - ASSERT(!new_object_failed); - PASS(); -} - -/* === Test adding an object without an associated task === */ - -void new_object_no_task_callback(ObjectID object_id, - TaskID task_id, - bool is_put, - void *user_context) { - new_object_succeeded = 1; - RAY_CHECK(task_id.is_nil()); - event_loop_stop(g_loop); -} - -TEST new_object_no_task_test(void) { - new_object_failed = 0; - new_object_succeeded = 0; - new_object_id = ObjectID::from_random(); - new_object_task_id = TaskID::from_random(); - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = new_object_fail_callback, - }; - result_table_lookup(db, new_object_id, &retry, new_object_no_task_callback, - NULL); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(new_object_succeeded); - ASSERT(!new_object_failed); - PASS(); -} - -/* ==== Test if operations time out correctly ==== */ - -/* === Test lookup timeout === */ - -const char *lookup_timeout_context = "lookup_timeout"; -int lookup_failed = 0; - -void lookup_done_callback(ObjectID object_id, - bool never_created, - const std::vector &manager_vector, - void *context) { - /* The done callback should not be called. */ - RAY_CHECK(0); -} - -void lookup_fail_callback(UniqueID id, void *user_context, void *user_data) { - lookup_failed = 1; - RAY_CHECK(user_context == (void *) lookup_timeout_context); - event_loop_stop(g_loop); -} - -TEST lookup_timeout_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, .timeout = 100, .fail_callback = lookup_fail_callback, - }; - object_table_lookup(db, UniqueID::nil(), &retry, lookup_done_callback, - (void *) lookup_timeout_context); - /* Disconnect the database to see if the lookup times out. */ - close(db->context->c.fd); - for (auto context : db->contexts) { - close(context->c.fd); - } - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(lookup_failed); - PASS(); -} - -/* === Test add timeout === */ - -const char *add_timeout_context = "add_timeout"; -int add_failed = 0; - -void add_done_callback(ObjectID object_id, bool success, void *user_context) { - /* The done callback should not be called. */ - RAY_CHECK(0); -} - -void add_fail_callback(UniqueID id, void *user_context, void *user_data) { - add_failed = 1; - RAY_CHECK(user_context == (void *) add_timeout_context); - event_loop_stop(g_loop); -} - -TEST add_timeout_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, .timeout = 100, .fail_callback = add_fail_callback, - }; - object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry, - add_done_callback, (void *) add_timeout_context); - /* Disconnect the database to see if the lookup times out. */ - close(db->context->c.fd); - for (auto context : db->contexts) { - close(context->c.fd); - } - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(add_failed); - PASS(); -} - -/* === Test subscribe timeout === */ - -int subscribe_failed = 0; - -void subscribe_done_callback(ObjectID object_id, - int64_t data_size, - const std::vector &manager_vector, - void *user_context) { - /* The done callback should not be called. */ - RAY_CHECK(0); -} - -void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) { - subscribe_failed = 1; - event_loop_stop(g_loop); -} - -TEST subscribe_timeout_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = subscribe_fail_callback, - }; - object_table_subscribe_to_notifications(db, false, subscribe_done_callback, - NULL, &retry, NULL, NULL); - /* Disconnect the database to see if the lookup times out. */ - close(db->subscribe_context->c.fd); - for (auto subscribe_context : db->subscribe_contexts) { - close(subscribe_context->c.fd); - } - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_failed); - PASS(); -} - -/* ==== Test if the retry is working correctly ==== */ - -int64_t reconnect_context_callback(event_loop *loop, - int64_t timer_id, - void *context) { - DBHandle *db = (DBHandle *) context; - /* Reconnect to redis. This is not reconnecting the pub/sub channel. */ - redisAsyncFree(db->context); - redisFree(db->sync_context); - db->context = redisAsyncConnect("127.0.0.1", 6379); - db->context->data = (void *) db; - db->sync_context = redisConnect("127.0.0.1", 6379); - /* Re-attach the database to the event loop (the file descriptor changed). */ - db_attach(db, loop, true); - RAY_LOG(DEBUG) << "Reconnected to Redis"; - return EVENT_LOOP_TIMER_DONE; -} - -int64_t terminate_event_loop_callback(event_loop *loop, - int64_t timer_id, - void *context) { - event_loop_stop(loop); - return EVENT_LOOP_TIMER_DONE; -} - -/* === Test lookup retry === */ - -const char *lookup_retry_context = "lookup_retry"; -int lookup_retry_succeeded = 0; - -void lookup_retry_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* The fail callback should not be called. */ - RAY_CHECK(0); -} - -/* === Test add retry === */ - -const char *add_retry_context = "add_retry"; -int add_retry_succeeded = 0; - -/* === Test add then lookup retry === */ - -void add_lookup_done_callback(ObjectID object_id, - bool never_created, - const std::vector &manager_ids, - void *context) { - DBHandle *db = (DBHandle *) context; - RAY_CHECK(manager_ids.size() == 1); - const std::vector managers = - db_client_table_get_ip_addresses(db, manager_ids); - RAY_CHECK(managers.at(0) == "127.0.0.1:11235"); - lookup_retry_succeeded = 1; -} - -void add_lookup_callback(ObjectID object_id, bool success, void *user_context) { - RAY_CHECK(success); - DBHandle *db = (DBHandle *) user_context; - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = lookup_retry_fail_callback, - }; - object_table_lookup(db, UniqueID::nil(), &retry, add_lookup_done_callback, - (void *) db); -} - -TEST add_lookup_test(void) { - g_loop = event_loop_create(); - lookup_retry_succeeded = 0; - /* Construct the arguments to db_connect. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:11235"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, g_loop, true); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = lookup_retry_fail_callback, - }; - object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry, - add_lookup_callback, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(lookup_retry_succeeded); - PASS(); -} - -/* === Test add, remove, then lookup === */ -void add_remove_lookup_done_callback( - ObjectID object_id, - bool never_created, - const std::vector &manager_vector, - void *context) { - RAY_CHECK(context == (void *) lookup_retry_context); - RAY_CHECK(manager_vector.size() == 0); - lookup_retry_succeeded = 1; -} - -void add_remove_lookup_callback(ObjectID object_id, - bool success, - void *user_context) { - RAY_CHECK(success); - DBHandle *db = (DBHandle *) user_context; - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = lookup_retry_fail_callback, - }; - object_table_lookup(db, UniqueID::nil(), &retry, - add_remove_lookup_done_callback, - (void *) lookup_retry_context); -} - -void add_remove_callback(ObjectID object_id, bool success, void *user_context) { - RAY_CHECK(success); - DBHandle *db = (DBHandle *) user_context; - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = lookup_retry_fail_callback, - }; - object_table_remove(db, UniqueID::nil(), NULL, &retry, - add_remove_lookup_callback, (void *) db); -} - -TEST add_remove_lookup_test(void) { - g_loop = event_loop_create(); - lookup_retry_succeeded = 0; - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, true); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = lookup_retry_fail_callback, - }; - object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry, - add_remove_callback, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(lookup_retry_succeeded); - PASS(); -} - -/* ==== Test if late succeed is working correctly ==== */ - -/* === Test lookup late succeed === */ - -const char *lookup_late_context = "lookup_late"; -int lookup_late_failed = 0; - -void lookup_late_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - RAY_CHECK(user_context == (void *) lookup_late_context); - lookup_late_failed = 1; -} - -void lookup_late_done_callback(ObjectID object_id, - bool never_created, - const std::vector &manager_vector, - void *context) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST lookup_late_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 0, - .timeout = 0, - .fail_callback = lookup_late_fail_callback, - }; - object_table_lookup(db, UniqueID::nil(), &retry, lookup_late_done_callback, - (void *) lookup_late_context); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* First process timer events to make sure the timeout is processed before - * anything else. */ - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(lookup_late_failed); - PASS(); -} - -/* === Test add late succeed === */ - -const char *add_late_context = "add_late"; -int add_late_failed = 0; - -void add_late_fail_callback(UniqueID id, void *user_context, void *user_data) { - RAY_CHECK(user_context == (void *) add_late_context); - add_late_failed = 1; -} - -void add_late_done_callback(ObjectID object_id, - bool success, - void *user_context) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST add_late_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 0, .timeout = 0, .fail_callback = add_late_fail_callback, - }; - object_table_add(db, UniqueID::nil(), 0, (unsigned char *) NIL_DIGEST, &retry, - add_late_done_callback, (void *) add_late_context); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* First process timer events to make sure the timeout is processed before - * anything else. */ - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(add_late_failed); - PASS(); -} - -/* === Test subscribe late succeed === */ - -const char *subscribe_late_context = "subscribe_late"; -int subscribe_late_failed = 0; - -void subscribe_late_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - RAY_CHECK(user_context == (void *) subscribe_late_context); - subscribe_late_failed = 1; -} - -void subscribe_late_done_callback(ObjectID object_id, - bool never_created, - const std::vector &manager_vector, - void *user_context) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST subscribe_late_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 0, - .timeout = 0, - .fail_callback = subscribe_late_fail_callback, - }; - object_table_subscribe_to_notifications(db, false, NULL, NULL, &retry, - subscribe_late_done_callback, - (void *) subscribe_late_context); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* First process timer events to make sure the timeout is processed before - * anything else. */ - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_late_failed); - PASS(); -} - -/* === Test subscribe object available succeed === */ - -const char *subscribe_success_context = "subscribe_success"; -int subscribe_success_done = 0; -int subscribe_success_succeeded = 0; -ObjectID subscribe_id; - -void subscribe_success_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -void subscribe_success_done_callback( - ObjectID object_id, - bool never_created, - const std::vector &manager_vector, - void *user_context) { - RetryInfo retry = { - .num_retries = 0, .timeout = 750, .fail_callback = NULL, - }; - object_table_add((DBHandle *) user_context, subscribe_id, 0, - (unsigned char *) NIL_DIGEST, &retry, NULL, NULL); - subscribe_success_done = 1; -} - -void subscribe_success_object_available_callback( - ObjectID object_id, - int64_t data_size, - const std::vector &manager_vector, - void *user_context) { - RAY_CHECK(user_context == (void *) subscribe_success_context); - RAY_CHECK(object_id == subscribe_id); - RAY_CHECK(manager_vector.size() == 1); - subscribe_success_succeeded = 1; -} - -TEST subscribe_success_test(void) { - g_loop = event_loop_create(); - - /* Construct the arguments to db_connect. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:11236"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, g_loop, false); - subscribe_id = ObjectID::from_random(); - - RetryInfo retry = { - .num_retries = 0, - .timeout = 100, - .fail_callback = subscribe_success_fail_callback, - }; - object_table_subscribe_to_notifications( - db, false, subscribe_success_object_available_callback, - (void *) subscribe_success_context, &retry, - subscribe_success_done_callback, (void *) db); - - ObjectID object_ids[1] = {subscribe_id}; - object_table_request_notifications(db, 1, object_ids, &retry); - - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - - ASSERT(subscribe_success_done); - ASSERT(subscribe_success_succeeded); - PASS(); -} - -/* Test if subscribe succeeds if the object is already present. */ -typedef struct { - const char *teststr; - int64_t data_size; -} subscribe_object_present_context_t; - -const char *subscribe_object_present_str = "subscribe_object_present"; -int subscribe_object_present_succeeded = 0; - -void subscribe_object_present_object_available_callback( - ObjectID object_id, - int64_t data_size, - const std::vector &manager_vector, - void *user_context) { - subscribe_object_present_context_t *ctx = - (subscribe_object_present_context_t *) user_context; - RAY_CHECK(ctx->data_size == data_size); - RAY_CHECK(strcmp(subscribe_object_present_str, ctx->teststr) == 0); - subscribe_object_present_succeeded = 1; - RAY_CHECK(manager_vector.size() == 1); -} - -void fatal_fail_callback(UniqueID id, void *user_context, void *user_data) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST subscribe_object_present_test(void) { - int64_t data_size = 0xF1F0; - subscribe_object_present_context_t myctx = {subscribe_object_present_str, - data_size}; - - g_loop = event_loop_create(); - /* Construct the arguments to db_connect. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:11236"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, g_loop, false); - UniqueID id = UniqueID::from_random(); - RetryInfo retry = { - .num_retries = 0, .timeout = 100, .fail_callback = fatal_fail_callback, - }; - object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry, - NULL, NULL); - object_table_subscribe_to_notifications( - db, false, subscribe_object_present_object_available_callback, - (void *) &myctx, &retry, NULL, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to create do the add and subscribe. */ - event_loop_run(g_loop); - - ObjectID object_ids[1] = {id}; - object_table_request_notifications(db, 1, object_ids, &retry); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the request notifications. */ - event_loop_run(g_loop); - - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_object_present_succeeded == 1); - PASS(); -} - -/* Test if subscribe is not called if object is not present. */ - -const char *subscribe_object_not_present_context = - "subscribe_object_not_present"; - -void subscribe_object_not_present_object_available_callback( - ObjectID object_id, - int64_t data_size, - const std::vector &manager_vector, - void *user_context) { - /* This should not be called. */ - RAY_CHECK(0); -} - -TEST subscribe_object_not_present_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - UniqueID id = UniqueID::from_random(); - RetryInfo retry = { - .num_retries = 0, .timeout = 100, .fail_callback = NULL, - }; - object_table_subscribe_to_notifications( - db, false, subscribe_object_not_present_object_available_callback, - (void *) subscribe_object_not_present_context, &retry, NULL, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the subscribe. */ - event_loop_run(g_loop); - - ObjectID object_ids[1] = {id}; - object_table_request_notifications(db, 1, object_ids, &retry); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the request notifications. */ - event_loop_run(g_loop); - - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - PASS(); -} - -/* Test if subscribe is called if object becomes available later. */ - -const char *subscribe_object_available_later_context = - "subscribe_object_available_later"; -int subscribe_object_available_later_succeeded = 0; - -void subscribe_object_available_later_object_available_callback( - ObjectID object_id, - int64_t data_size, - const std::vector &manager_vector, - void *user_context) { - subscribe_object_present_context_t *myctx = - (subscribe_object_present_context_t *) user_context; - RAY_CHECK(myctx->data_size == data_size); - RAY_CHECK(strcmp(myctx->teststr, subscribe_object_available_later_context) == - 0); - /* Make sure the callback is only called once. */ - subscribe_object_available_later_succeeded += 1; - RAY_CHECK(manager_vector.size() == 1); -} - -TEST subscribe_object_available_later_test(void) { - int64_t data_size = 0xF1F0; - subscribe_object_present_context_t *myctx = - (subscribe_object_present_context_t *) malloc( - sizeof(subscribe_object_present_context_t)); - myctx->teststr = subscribe_object_available_later_context; - myctx->data_size = data_size; - - g_loop = event_loop_create(); - /* Construct the arguments to db_connect. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:11236"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, g_loop, false); - UniqueID id = UniqueID::from_random(); - RetryInfo retry = { - .num_retries = 0, .timeout = 100, .fail_callback = NULL, - }; - object_table_subscribe_to_notifications( - db, false, subscribe_object_available_later_object_available_callback, - (void *) myctx, &retry, NULL, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the subscribe. */ - event_loop_run(g_loop); - - ObjectID object_ids[1] = {id}; - object_table_request_notifications(db, 1, object_ids, &retry); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the request notifications. */ - event_loop_run(g_loop); - - ASSERT_EQ(subscribe_object_available_later_succeeded, 0); - object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry, - NULL, NULL); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the object table add. */ - event_loop_run(g_loop); - - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT_EQ(subscribe_object_available_later_succeeded, 1); - /* Reset the global variable before exiting this unit test. */ - subscribe_object_available_later_succeeded = 0; - free(myctx); - PASS(); -} - -TEST subscribe_object_available_subscribe_all(void) { - int64_t data_size = 0xF1F0; - subscribe_object_present_context_t myctx = { - subscribe_object_available_later_context, data_size}; - g_loop = event_loop_create(); - /* Construct the arguments to db_connect. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:11236"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, g_loop, false); - UniqueID id = UniqueID::from_random(); - RetryInfo retry = { - .num_retries = 0, .timeout = 100, .fail_callback = NULL, - }; - object_table_subscribe_to_notifications( - db, true, subscribe_object_available_later_object_available_callback, - (void *) &myctx, &retry, NULL, (void *) db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the subscribe. */ - event_loop_run(g_loop); - - /* At this point we don't expect any object notifications received. */ - ASSERT_EQ(subscribe_object_available_later_succeeded, 0); - object_table_add(db, id, data_size, (unsigned char *) NIL_DIGEST, &retry, - NULL, NULL); - /* Install handler to terminate event loop after 750ms. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* Run the event loop to do the object table add. */ - event_loop_run(g_loop); - /* At this point we assume that object table add completed. */ - - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - /* Assert that the object table add completed and notification callback fired. - */ - printf("subscribe_all object info test: callback fired: %d times\n", - subscribe_object_available_later_succeeded); - fflush(stdout); - ASSERT_EQ(subscribe_object_available_later_succeeded, 1); - /* Reset the global variable before exiting this unit test. */ - subscribe_object_available_later_succeeded = 0; - PASS(); -} - -SUITE(object_table_tests) { - RUN_REDIS_TEST(new_object_test); - RUN_REDIS_TEST(new_object_no_task_test); - // RUN_REDIS_TEST(lookup_timeout_test); - // RUN_REDIS_TEST(add_timeout_test); - // RUN_REDIS_TEST(subscribe_timeout_test); - RUN_REDIS_TEST(add_lookup_test); - RUN_REDIS_TEST(add_remove_lookup_test); - // RUN_REDIS_TEST(lookup_late_test); - // RUN_REDIS_TEST(add_late_test); - // RUN_REDIS_TEST(subscribe_late_test); - RUN_REDIS_TEST(subscribe_success_test); - RUN_REDIS_TEST(subscribe_object_not_present_test); - RUN_REDIS_TEST(subscribe_object_available_later_test); - RUN_REDIS_TEST(subscribe_object_available_subscribe_all); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - g_task_builder = make_task_builder(); - GREATEST_MAIN_BEGIN(); - RUN_SUITE(object_table_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/redis_tests.cc b/src/common/test/redis_tests.cc deleted file mode 100644 index 7db7ae2ee..000000000 --- a/src/common/test/redis_tests.cc +++ /dev/null @@ -1,238 +0,0 @@ -#include "greatest.h" - -#include -#include - -#include - -#include "event_loop.h" -#include "state/db.h" -#include "state/redis.h" -#include "io.h" -#include "logging.h" -#include "test_common.h" - -SUITE(redis_tests); - -const char *test_set_format = "SET %s %s"; -const char *test_get_format = "GET %s"; -const char *test_key = "foo"; -const char *test_value = "bar"; -std::vector connections; - -void write_formatted_log_message(int socket_fd, const char *format, ...) { - va_list ap; - - /* Get cmd size */ - va_start(ap, format); - size_t cmd_size = vsnprintf(nullptr, 0, format, ap) + 1; - va_end(ap); - - /* Print va to cmd */ - char cmd[cmd_size]; - va_start(ap, format); - vsnprintf(cmd, cmd_size, format, ap); - va_end(ap); - - write_log_message(socket_fd, cmd); -} - -int async_redis_socket_test_callback_called = 0; - -void async_redis_socket_test_callback(redisAsyncContext *ac, - void *r, - void *privdata) { - async_redis_socket_test_callback_called = 1; - redisContext *context = redisConnect("127.0.0.1", 6379); - redisReply *reply = - (redisReply *) redisCommand(context, test_get_format, test_key); - redisFree(context); - RAY_CHECK(reply != NULL); - if (strcmp(reply->str, test_value)) { - freeReplyObject(reply); - RAY_CHECK(0); - } - freeReplyObject(reply); -} - -TEST redis_socket_test(void) { - const char *socket_pathname = "/tmp/redis-test-socket"; - redisContext *context = redisConnect("127.0.0.1", 6379); - ASSERT(context != NULL); - int socket_fd = bind_ipc_sock(socket_pathname, true); - ASSERT(socket_fd >= 0); - - int client_fd = connect_ipc_sock(socket_pathname); - ASSERT(client_fd >= 0); - write_formatted_log_message(client_fd, test_set_format, test_key, test_value); - - int server_fd = accept_client(socket_fd); - char *cmd = read_log_message(server_fd); - close(client_fd); - close(server_fd); - close(socket_fd); - unlink(socket_pathname); - - redisReply *reply = (redisReply *) redisCommand(context, cmd, 0, 0); - freeReplyObject(reply); - reply = (redisReply *) redisCommand(context, "GET %s", test_key); - ASSERT(reply != NULL); - ASSERT_STR_EQ(reply->str, test_value); - freeReplyObject(reply); - - free(cmd); - redisFree(context); - PASS(); -} - -void redis_read_callback(event_loop *loop, int fd, void *context, int events) { - DBHandle *db = (DBHandle *) context; - char *cmd = read_log_message(fd); - redisAsyncCommand(db->context, async_redis_socket_test_callback, NULL, cmd); - free(cmd); -} - -void redis_accept_callback(event_loop *loop, - int socket_fd, - void *context, - int events) { - int accept_fd = accept_client(socket_fd); - RAY_CHECK(accept_fd >= 0); - connections.push_back(accept_fd); - event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, redis_read_callback, - context); -} - -int timeout_handler(event_loop *loop, timer_id timer_id, void *context) { - event_loop_stop(loop); - return EVENT_LOOP_TIMER_DONE; -} - -TEST async_redis_socket_test(void) { - event_loop *loop = event_loop_create(); - - /* Start IPC channel. */ - const char *socket_pathname = "/tmp/async-redis-test-socket"; - int socket_fd = bind_ipc_sock(socket_pathname, true); - ASSERT(socket_fd >= 0); - connections.push_back(socket_fd); - - /* Start connection to Redis. */ - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "test_process", - "127.0.0.1", std::vector()); - db_attach(db, loop, false); - - /* Send a command to the Redis process. */ - int client_fd = connect_ipc_sock(socket_pathname); - ASSERT(client_fd >= 0); - connections.push_back(client_fd); - write_formatted_log_message(client_fd, test_set_format, test_key, test_value); - - event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, redis_read_callback, - db); - event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, redis_accept_callback, - db); - event_loop_add_timer(loop, 100, timeout_handler, NULL); - event_loop_run(loop); - - ASSERT(async_redis_socket_test_callback_called); - - db_disconnect(db); - event_loop_destroy(loop); - - for (int const &p : connections) { - close(p); - } - unlink(socket_pathname); - connections.clear(); - PASS(); -} - -int logging_test_callback_called = 0; - -void logging_test_callback(redisAsyncContext *ac, void *r, void *privdata) { - logging_test_callback_called = 1; - redisContext *context = redisConnect("127.0.0.1", 6379); - redisReply *reply = (redisReply *) redisCommand(context, "KEYS %s", "log:*"); - redisFree(context); - RAY_CHECK(reply != NULL); - RAY_CHECK(reply->elements > 0); - freeReplyObject(reply); -} - -void logging_read_callback(event_loop *loop, - int fd, - void *context, - int events) { - DBHandle *conn = (DBHandle *) context; - char *cmd = read_log_message(fd); - redisAsyncCommand(conn->context, logging_test_callback, NULL, cmd, - (char *) conn->client.data(), sizeof(conn->client)); - free(cmd); -} - -void logging_accept_callback(event_loop *loop, - int socket_fd, - void *context, - int events) { - int accept_fd = accept_client(socket_fd); - RAY_CHECK(accept_fd >= 0); - connections.push_back(accept_fd); - event_loop_add_file(loop, accept_fd, EVENT_LOOP_READ, logging_read_callback, - context); -} - -TEST logging_test(void) { - event_loop *loop = event_loop_create(); - - /* Start IPC channel. */ - const char *socket_pathname = "/tmp/logging-test-socket"; - int socket_fd = bind_ipc_sock(socket_pathname, true); - ASSERT(socket_fd >= 0); - connections.push_back(socket_fd); - - /* Start connection to Redis. */ - DBHandle *conn = db_connect(std::string("127.0.0.1"), 6379, "test_process", - "127.0.0.1", std::vector()); - db_attach(conn, loop, false); - - /* Send a command to the Redis process. */ - int client_fd = connect_ipc_sock(socket_pathname); - ASSERT(client_fd >= 0); - connections.push_back(client_fd); - RayLogger *logger = RayLogger_init("worker", RAY_LOG_INFO, 0, &client_fd); - RayLogger_log(logger, RAY_LOG_INFO, "TEST", "Message"); - - event_loop_add_file(loop, socket_fd, EVENT_LOOP_READ, logging_accept_callback, - conn); - event_loop_add_file(loop, client_fd, EVENT_LOOP_READ, logging_read_callback, - conn); - event_loop_add_timer(loop, 100, timeout_handler, NULL); - event_loop_run(loop); - - ASSERT(logging_test_callback_called); - - RayLogger_free(logger); - db_disconnect(conn); - event_loop_destroy(loop); - for (int const &p : connections) { - close(p); - } - unlink(socket_pathname); - connections.clear(); - PASS(); -} - -SUITE(redis_tests) { - RUN_REDIS_TEST(redis_socket_test); - RUN_REDIS_TEST(async_redis_socket_test); - RUN_REDIS_TEST(logging_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); - RUN_SUITE(redis_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/run_tests.sh b/src/common/test/run_tests.sh deleted file mode 100644 index 5ccb1e3f9..000000000 --- a/src/common/test/run_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# This needs to be run in the build tree, which is normally ray/build - -# Cause the script to exit if a single command fails. -set -ex - -LaunchRedis() { - port=$1 - if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then - ./src/credis/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/credis/build/src/libmember.so \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - else - ./src/common/thirdparty/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - fi - sleep 1s -} - - -# Start the Redis shards. -LaunchRedis 6379 -LaunchRedis 6380 -# Register the shard location with the primary shard. -./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 -./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 - -if [ -z "$RAY_USE_NEW_GCS" ]; then - ./src/common/db_tests - ./src/common/io_tests - ./src/common/task_tests - ./src/common/redis_tests - ./src/common/task_table_tests - ./src/common/object_table_tests -fi - -./src/common/thirdparty/redis/src/redis-cli -p 6379 shutdown -./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown diff --git a/src/common/test/run_valgrind.sh b/src/common/test/run_valgrind.sh deleted file mode 100644 index 418a91366..000000000 --- a/src/common/test/run_valgrind.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# This needs to be run in the build tree, which is normally ray/build - -set -x - -# Cause the script to exit if a single command fails. -set -e - -if [ -z "$RAY_USE_NEW_GCS" ]; then - # Start the Redis shards. - ./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 & - ./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 & - sleep 1s - # Register the shard location with the primary shard. - ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 - ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 - - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/db_tests - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/io_tests - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/task_tests - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/redis_tests - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/task_table_tests - valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/common/object_table_tests - ./src/common/thirdparty/redis/src/redis-cli shutdown - ./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown -fi diff --git a/src/common/test/task_table_tests.cc b/src/common/test/task_table_tests.cc deleted file mode 100644 index f94aca3b1..000000000 --- a/src/common/test/task_table_tests.cc +++ /dev/null @@ -1,460 +0,0 @@ -#include "greatest.h" - -#include "event_loop.h" -#include "example_task.h" -#include "test_common.h" -#include "common.h" -#include "state/object_table.h" -#include "state/redis.h" - -#include -#include - -SUITE(task_table_tests); - -event_loop *g_loop; -TaskBuilder *g_task_builder = NULL; - -/* ==== Test operations in non-failure scenario ==== */ - -/* === A lookup of a task not in the table === */ - -TaskID lookup_nil_id; -int lookup_nil_success = 0; -const char *lookup_nil_context = "lookup_nil"; - -void lookup_nil_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* The fail callback should not be called. */ - RAY_CHECK(0); -} - -void lookup_nil_success_callback(Task *task, void *context) { - lookup_nil_success = 1; - RAY_CHECK(task == NULL); - RAY_CHECK(context == (void *) lookup_nil_context); - event_loop_stop(g_loop); -} - -TEST lookup_nil_test(void) { - lookup_nil_id = TaskID::from_random(); - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 1000, - .fail_callback = lookup_nil_fail_callback, - }; - task_table_get_task(db, lookup_nil_id, &retry, lookup_nil_success_callback, - (void *) lookup_nil_context); - /* Disconnect the database to see if the lookup times out. */ - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(lookup_nil_success); - PASS(); -} - -/* === A lookup of a task after it's added returns the same spec === */ - -int add_success = 0; -int lookup_success = 0; -Task *add_lookup_task; -const char *add_lookup_context = "add_lookup"; - -void add_lookup_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* The fail callback should not be called. */ - RAY_CHECK(0); -} - -void lookup_success_callback(Task *task, void *context) { - lookup_success = 1; - RAY_CHECK(Task_equals(task, add_lookup_task)); - event_loop_stop(g_loop); -} - -void add_success_callback(TaskID task_id, void *context) { - add_success = 1; - RAY_CHECK(TaskID_equal(task_id, Task_task_id(add_lookup_task))); - - DBHandle *db = (DBHandle *) context; - RetryInfo retry = { - .num_retries = 5, - .timeout = 1000, - .fail_callback = add_lookup_fail_callback, - }; - task_table_get_task(db, task_id, &retry, lookup_success_callback, - (void *) add_lookup_context); -} - -void subscribe_success_callback(TaskID task_id, void *context) { - DBHandle *db = (DBHandle *) context; - RetryInfo retry = { - .num_retries = 5, - .timeout = 1000, - .fail_callback = add_lookup_fail_callback, - }; - task_table_add_task(db, Task_copy(add_lookup_task), &retry, - add_success_callback, (void *) db); -} - -TEST add_lookup_test(void) { - add_lookup_task = example_task(1, 1, TaskStatus::WAITING); - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 1000, - .fail_callback = add_lookup_fail_callback, - }; - /* Wait for subscription to succeed before adding the task. */ - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, subscribe_success_callback, (void *) db); - /* Disconnect the database to see if the lookup times out. */ - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(add_success); - ASSERT(lookup_success); - PASS(); -} - -/* ==== Test if operations time out correctly ==== */ - -/* === Test subscribe timeout === */ - -const char *subscribe_timeout_context = "subscribe_timeout"; -int subscribe_failed = 0; - -void subscribe_done_callback(TaskID task_id, void *user_context) { - /* The done callback should not be called. */ - RAY_CHECK(0); -} - -void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) { - subscribe_failed = 1; - RAY_CHECK(user_context == (void *) subscribe_timeout_context); - event_loop_stop(g_loop); -} - -TEST subscribe_timeout_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = subscribe_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, subscribe_done_callback, - (void *) subscribe_timeout_context); - /* Disconnect the database to see if the subscribe times out. */ - close(db->subscribe_context->c.fd); - for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) { - close(db->subscribe_contexts[i]->c.fd); - } - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_failed); - PASS(); -} - -/* === Test publish timeout === */ - -const char *publish_timeout_context = "publish_timeout"; -int publish_failed = 0; - -void publish_done_callback(TaskID task_id, void *user_context) { - /* The done callback should not be called. */ - RAY_CHECK(0); -} - -void publish_fail_callback(UniqueID id, void *user_context, void *user_data) { - publish_failed = 1; - RAY_CHECK(user_context == (void *) publish_timeout_context); - event_loop_stop(g_loop); -} - -TEST publish_timeout_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - Task *task = example_task(1, 1, TaskStatus::WAITING); - RetryInfo retry = { - .num_retries = 5, .timeout = 100, .fail_callback = publish_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, NULL, NULL); - task_table_add_task(db, task, &retry, publish_done_callback, - (void *) publish_timeout_context); - /* Disconnect the database to see if the publish times out. */ - close(db->context->c.fd); - for (size_t i = 0; i < db->contexts.size(); ++i) { - close(db->contexts[i]->c.fd); - } - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(publish_failed); - PASS(); -} - -/* ==== Test if the retry is working correctly ==== */ - -int64_t reconnect_db_callback(event_loop *loop, - int64_t timer_id, - void *context) { - DBHandle *db = (DBHandle *) context; - /* Reconnect to redis. */ - redisAsyncFree(db->subscribe_context); - db->subscribe_context = redisAsyncConnect("127.0.0.1", 6379); - db->subscribe_context->data = (void *) db; - for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) { - redisAsyncFree(db->subscribe_contexts[i]); - db->subscribe_contexts[i] = redisAsyncConnect("127.0.0.1", 6380 + i); - db->subscribe_contexts[i]->data = (void *) db; - } - /* Re-attach the database to the event loop (the file descriptor changed). */ - db_attach(db, loop, true); - return EVENT_LOOP_TIMER_DONE; -} - -int64_t terminate_event_loop_callback(event_loop *loop, - int64_t timer_id, - void *context) { - event_loop_stop(loop); - return EVENT_LOOP_TIMER_DONE; -} - -/* === Test subscribe retry === */ - -const char *subscribe_retry_context = "subscribe_retry"; -int subscribe_retry_succeeded = 0; - -void subscribe_retry_done_callback(ObjectID object_id, void *user_context) { - RAY_CHECK(user_context == (void *) subscribe_retry_context); - subscribe_retry_succeeded = 1; -} - -void subscribe_retry_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* The fail callback should not be called. */ - RAY_CHECK(0); -} - -TEST subscribe_retry_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = subscribe_retry_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, subscribe_retry_done_callback, - (void *) subscribe_retry_context); - /* Disconnect the database to see if the subscribe times out. */ - close(db->subscribe_context->c.fd); - for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) { - close(db->subscribe_contexts[i]->c.fd); - } - /* Install handler for reconnecting the database. */ - event_loop_add_timer(g_loop, 150, - (event_loop_timer_handler) reconnect_db_callback, db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_retry_succeeded); - PASS(); -} - -/* === Test publish retry === */ - -const char *publish_retry_context = "publish_retry"; -int publish_retry_succeeded = 0; - -void publish_retry_done_callback(ObjectID object_id, void *user_context) { - RAY_CHECK(user_context == (void *) publish_retry_context); - publish_retry_succeeded = 1; -} - -void publish_retry_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - /* The fail callback should not be called. */ - RAY_CHECK(0); -} - -TEST publish_retry_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - Task *task = example_task(1, 1, TaskStatus::WAITING); - RetryInfo retry = { - .num_retries = 5, - .timeout = 100, - .fail_callback = publish_retry_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, NULL, NULL); - task_table_add_task(db, task, &retry, publish_retry_done_callback, - (void *) publish_retry_context); - /* Disconnect the database to see if the publish times out. */ - close(db->subscribe_context->c.fd); - for (size_t i = 0; i < db->subscribe_contexts.size(); ++i) { - close(db->subscribe_contexts[i]->c.fd); - } - /* Install handler for reconnecting the database. */ - event_loop_add_timer(g_loop, 150, - (event_loop_timer_handler) reconnect_db_callback, db); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(publish_retry_succeeded); - PASS(); -} - -/* ==== Test if late succeed is working correctly ==== */ - -/* === Test subscribe late succeed === */ - -const char *subscribe_late_context = "subscribe_late"; -int subscribe_late_failed = 0; - -void subscribe_late_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - RAY_CHECK(user_context == (void *) subscribe_late_context); - subscribe_late_failed = 1; -} - -void subscribe_late_done_callback(TaskID task_id, void *user_context) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST subscribe_late_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - RetryInfo retry = { - .num_retries = 0, - .timeout = 0, - .fail_callback = subscribe_late_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - &retry, subscribe_late_done_callback, - (void *) subscribe_late_context); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* First process timer events to make sure the timeout is processed before - * anything else. */ - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(subscribe_late_failed); - PASS(); -} - -/* === Test publish late succeed === */ - -const char *publish_late_context = "publish_late"; -int publish_late_failed = 0; - -void publish_late_fail_callback(UniqueID id, - void *user_context, - void *user_data) { - RAY_CHECK(user_context == (void *) publish_late_context); - publish_late_failed = 1; -} - -void publish_late_done_callback(TaskID task_id, void *user_context) { - /* This function should never be called. */ - RAY_CHECK(0); -} - -TEST publish_late_test(void) { - g_loop = event_loop_create(); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", std::vector()); - db_attach(db, g_loop, false); - Task *task = example_task(1, 1, TaskStatus::WAITING); - RetryInfo retry = { - .num_retries = 0, - .timeout = 0, - .fail_callback = publish_late_fail_callback, - }; - task_table_subscribe(db, UniqueID::nil(), TaskStatus::WAITING, NULL, NULL, - NULL, NULL, NULL); - task_table_add_task(db, task, &retry, publish_late_done_callback, - (void *) publish_late_context); - /* Install handler for terminating the event loop. */ - event_loop_add_timer(g_loop, 750, - (event_loop_timer_handler) terminate_event_loop_callback, - NULL); - /* First process timer events to make sure the timeout is processed before - * anything else. */ - aeProcessEvents(g_loop, AE_TIME_EVENTS); - event_loop_run(g_loop); - db_disconnect(db); - destroy_outstanding_callbacks(g_loop); - event_loop_destroy(g_loop); - ASSERT(publish_late_failed); - PASS(); -} - -SUITE(task_table_tests) { - RUN_REDIS_TEST(lookup_nil_test); - RUN_REDIS_TEST(add_lookup_test); - // RUN_REDIS_TEST(subscribe_timeout_test); - // RUN_REDIS_TEST(publish_timeout_test); - // RUN_REDIS_TEST(subscribe_retry_test); - // RUN_REDIS_TEST(publish_retry_test); - // RUN_REDIS_TEST(subscribe_late_test); - // RUN_REDIS_TEST(publish_late_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - g_task_builder = make_task_builder(); - GREATEST_MAIN_BEGIN(); - RUN_SUITE(task_table_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/task_tests.cc b/src/common/test/task_tests.cc deleted file mode 100644 index 2277912e7..000000000 --- a/src/common/test/task_tests.cc +++ /dev/null @@ -1,212 +0,0 @@ -#include "greatest.h" - -#include -#include -#include - -#include "common.h" -#include "test_common.h" -#include "task.h" -#include "io.h" - -SUITE(task_tests); - -TEST task_test(void) { - TaskID parent_task_id = TaskID::from_random(); - FunctionID func_id = FunctionID::from_random(); - TaskBuilder *builder = make_task_builder(); - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 2); - - UniqueID arg1 = UniqueID::from_random(); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, (uint8_t *) "hello", 5); - UniqueID arg2 = UniqueID::from_random(); - TaskSpec_args_add_ref(builder, &arg2, 1); - TaskSpec_args_add_val(builder, (uint8_t *) "world", 5); - /* Finish constructing the spec. This constructs the task ID and the - * return IDs. */ - int64_t size; - TaskSpec *spec = TaskSpec_finish_construct(builder, &size); - - /* Check that the spec was constructed as expected. */ - ASSERT(TaskSpec_num_args(spec) == 4); - ASSERT(TaskSpec_num_returns(spec) == 2); - ASSERT(FunctionID_equal(TaskSpec_function(spec), func_id)); - ASSERT(TaskSpec_arg_id(spec, 0, 0) == arg1); - ASSERT(memcmp(TaskSpec_arg_val(spec, 1), (uint8_t *) "hello", - TaskSpec_arg_length(spec, 1)) == 0); - ASSERT(TaskSpec_arg_id(spec, 2, 0) == arg2); - ASSERT(memcmp(TaskSpec_arg_val(spec, 3), (uint8_t *) "world", - TaskSpec_arg_length(spec, 3)) == 0); - - TaskSpec_free(spec); - free_task_builder(builder); - PASS(); -} - -TEST deterministic_ids_test(void) { - TaskBuilder *builder = make_task_builder(); - /* Define the inputs to the task construction. */ - TaskID parent_task_id = TaskID::from_random(); - FunctionID func_id = FunctionID::from_random(); - UniqueID arg1 = UniqueID::from_random(); - uint8_t *arg2 = (uint8_t *) "hello world"; - - /* Construct a first task. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size1; - TaskSpec *spec1 = TaskSpec_finish_construct(builder, &size1); - - /* Construct a second identical task. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size2; - TaskSpec *spec2 = TaskSpec_finish_construct(builder, &size2); - - /* Check that these tasks have the same task IDs and the same return IDs. */ - ASSERT(TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec2))); - ASSERT(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 0)); - ASSERT(TaskSpec_return(spec1, 1) == TaskSpec_return(spec2, 1)); - ASSERT(TaskSpec_return(spec1, 2) == TaskSpec_return(spec2, 2)); - /* Check that the return IDs are all distinct. */ - ASSERT(!(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 1))); - ASSERT(!(TaskSpec_return(spec1, 0) == TaskSpec_return(spec2, 2))); - ASSERT(!(TaskSpec_return(spec1, 1) == TaskSpec_return(spec2, 2))); - - /* Create more tasks that are only mildly different. */ - - /* Construct a task with a different parent task ID. */ - TaskSpec_start_construct(builder, DriverID::nil(), TaskID::from_random(), 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size3; - TaskSpec *spec3 = TaskSpec_finish_construct(builder, &size3); - - /* Construct a task with a different parent counter. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 1, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size4; - TaskSpec *spec4 = TaskSpec_finish_construct(builder, &size4); - - /* Construct a task with a different function ID. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, FunctionID::from_random(), - 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size5; - TaskSpec *spec5 = TaskSpec_finish_construct(builder, &size5); - - /* Construct a task with a different object ID argument. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - ObjectID object_id = ObjectID::from_random(); - TaskSpec_args_add_ref(builder, &object_id, 1); - TaskSpec_args_add_val(builder, arg2, 11); - int64_t size6; - TaskSpec *spec6 = TaskSpec_finish_construct(builder, &size6); - - /* Construct a task with a different value argument. */ - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 3); - TaskSpec_args_add_ref(builder, &arg1, 1); - TaskSpec_args_add_val(builder, (uint8_t *) "hello_world", 11); - int64_t size7; - TaskSpec *spec7 = TaskSpec_finish_construct(builder, &size7); - - /* Check that the task IDs are all distinct from the original. */ - ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec3))); - ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec4))); - ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec5))); - ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec6))); - ASSERT(!TaskID_equal(TaskSpec_task_id(spec1), TaskSpec_task_id(spec7))); - - /* Check that the return object IDs are distinct from the originals. */ - TaskSpec *specs[6] = {spec1, spec3, spec4, spec5, spec6, spec7}; - for (int task_index1 = 0; task_index1 < 6; ++task_index1) { - for (int return_index1 = 0; return_index1 < 3; ++return_index1) { - for (int task_index2 = 0; task_index2 < 6; ++task_index2) { - for (int return_index2 = 0; return_index2 < 3; ++return_index2) { - if (task_index1 != task_index2 && return_index1 != return_index2) { - ASSERT(!(TaskSpec_return(specs[task_index1], return_index1) == - TaskSpec_return(specs[task_index2], return_index2))); - } - } - } - } - } - - TaskSpec_free(spec1); - TaskSpec_free(spec2); - TaskSpec_free(spec3); - TaskSpec_free(spec4); - TaskSpec_free(spec5); - TaskSpec_free(spec6); - TaskSpec_free(spec7); - free_task_builder(builder); - PASS(); -} - -TEST send_task(void) { - TaskBuilder *builder = make_task_builder(); - TaskID parent_task_id = TaskID::from_random(); - FunctionID func_id = FunctionID::from_random(); - TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0, - ActorID::nil(), ObjectID::nil(), ActorID::nil(), - ActorID::nil(), 0, false, func_id, 2); - ObjectID object_id = ObjectID::from_random(); - TaskSpec_args_add_ref(builder, &object_id, 1); - TaskSpec_args_add_val(builder, (uint8_t *) "Hello", 5); - TaskSpec_args_add_val(builder, (uint8_t *) "World", 5); - object_id = ObjectID::from_random(); - TaskSpec_args_add_ref(builder, &object_id, 1); - int64_t size; - TaskSpec *spec = TaskSpec_finish_construct(builder, &size); - int fd[2]; - socketpair(AF_UNIX, SOCK_STREAM, 0, fd); - write_message(fd[0], static_cast(CommonMessageType::SUBMIT_TASK), - size, (uint8_t *) spec); - int64_t type; - int64_t length; - uint8_t *message; - read_message(fd[1], &type, &length, &message); - TaskSpec *result = (TaskSpec *) message; - ASSERT(static_cast(type) == - CommonMessageType::SUBMIT_TASK); - ASSERT(memcmp(spec, result, size) == 0); - TaskSpec_free(spec); - free(result); - free_task_builder(builder); - PASS(); -} - -SUITE(task_tests) { - RUN_TEST(task_test); - RUN_TEST(deterministic_ids_test); - RUN_TEST(send_task); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); - RUN_SUITE(task_tests); - GREATEST_MAIN_END(); -} diff --git a/src/common/test/test_common.h b/src/common/test/test_common.h deleted file mode 100644 index 03984e6f2..000000000 --- a/src/common/test/test_common.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef TEST_COMMON_H -#define TEST_COMMON_H - -#include - -#include -#include -#include - -#include "common.h" -#include "io.h" -#include "hiredis/hiredis.h" -#include "state/redis.h" - -#ifndef _WIN32 -/* This function is actually not declared in standard POSIX, so declare it. */ -extern int usleep(useconds_t usec); -#endif - -/* I/O helper methods to retry binding to sockets. */ -static inline std::string bind_ipc_sock_retry(const char *socket_name_format, - int *fd) { - std::string socket_name; - for (int num_retries = 0; num_retries < 5; ++num_retries) { - RAY_LOG(INFO) << "trying to find plasma socket (attempt " << num_retries - << ")"; - size_t size = std::snprintf(nullptr, 0, socket_name_format, rand()) + 1; - char socket_name_c_str[size]; - std::snprintf(socket_name_c_str, size, socket_name_format, rand()); - socket_name = std::string(socket_name_c_str); - - *fd = bind_ipc_sock(socket_name.c_str(), true); - if (*fd < 0) { - /* Sleep for 100ms. */ - usleep(100000); - continue; - } - break; - } - return socket_name; -} - -static inline int bind_inet_sock_retry(int *fd) { - int port = -1; - for (int num_retries = 0; num_retries < 5; ++num_retries) { - port = 10000 + rand() % 40000; - *fd = bind_inet_sock(port, true); - if (*fd < 0) { - /* Sleep for 100ms. */ - usleep(100000); - continue; - } - break; - } - return port; -} - -/* Flush redis. */ -static inline void flushall_redis(void) { - /* Flush the primary shard. */ - redisContext *context = redisConnect("127.0.0.1", 6379); - std::vector db_shards_addresses; - std::vector db_shards_ports; - get_redis_shards(context, db_shards_addresses, db_shards_ports); - freeReplyObject(redisCommand(context, "FLUSHALL")); - /* Readd the shard locations. */ - freeReplyObject(redisCommand(context, "SET NumRedisShards %d", - db_shards_addresses.size())); - for (size_t i = 0; i < db_shards_addresses.size(); ++i) { - freeReplyObject(redisCommand(context, "RPUSH RedisShards %s:%d", - db_shards_addresses[i].c_str(), - db_shards_ports[i])); - } - redisFree(context); - - /* Flush the remaining shards. */ - for (size_t i = 0; i < db_shards_addresses.size(); ++i) { - context = redisConnect(db_shards_addresses[i].c_str(), db_shards_ports[i]); - freeReplyObject(redisCommand(context, "FLUSHALL")); - redisFree(context); - } -} - -/* Cleanup method for running tests with the greatest library. - * Runs the test, then clears the Redis database. */ -#define RUN_REDIS_TEST(test) \ - flushall_redis(); \ - RUN_TEST(test); \ - flushall_redis(); - -#endif /* TEST_COMMON */ diff --git a/src/common/thirdparty/download_thirdparty.bat b/src/common/thirdparty/download_thirdparty.bat deleted file mode 100644 index 988592f83..000000000 --- a/src/common/thirdparty/download_thirdparty.bat +++ /dev/null @@ -1,15 +0,0 @@ -@SetLocal - @Echo Off - @PushD "%~dp0" - git submodule update --init --jobs="%NUMBER_OF_PROCESSORS%" - @If Not Exist "python\.git" git clone "https://github.com/austinsc/python.git" - Call :GitApply "python" "%CD%/patches/windows/python-pyconfig.patch" - Call :GitApply "redis-windows" "%CD%/patches/windows/redis.patch" - @PopD -@EndLocal -@GoTo :EOF - -:GitApply - @REM Check if patch already applied by attempting to apply it in reverse; if not, then force-reapply it - git -C "%~1" apply "%~2" -R --check 2> NUL || git -C "%~1" apply "%~2" --3way 2> NUL || git -C "%~1" reset --hard && git -C "%~1" apply "%~2" --3way -@GoTo :EOF diff --git a/src/common/thirdparty/greatest.h b/src/common/thirdparty/greatest.h deleted file mode 100644 index eb34ff426..000000000 --- a/src/common/thirdparty/greatest.h +++ /dev/null @@ -1,1023 +0,0 @@ -/* - * Copyright (c) 2011-2016 Scott Vokes - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef GREATEST_H -#define GREATEST_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* 1.2.1 */ -#define GREATEST_VERSION_MAJOR 1 -#define GREATEST_VERSION_MINOR 2 -#define GREATEST_VERSION_PATCH 1 - -/* A unit testing system for C, contained in 1 file. - * It doesn't use dynamic allocation or depend on anything - * beyond ANSI C89. - * - * An up-to-date version can be found at: - * https://github.com/silentbicycle/greatest/ - */ - - -/********************************************************************* - * Minimal test runner template - *********************************************************************/ -#if 0 -#include "greatest.h" -TEST foo_should_foo(void) { - PASS(); -} -static void setup_cb(void *data) { - printf("setup callback for each test case\n"); -} -static void teardown_cb(void *data) { - printf("teardown callback for each test case\n"); -} -SUITE(suite) { - /* Optional setup/teardown callbacks which will be run before/after - * every test case. If using a test suite, they will be cleared when - * the suite finishes. */ - SET_SETUP(setup_cb, voidp_to_callback_data); - SET_TEARDOWN(teardown_cb, voidp_to_callback_data); - RUN_TEST(foo_should_foo); -} -/* Add definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); -/* Set up, run suite(s) of tests, report pass/fail/skip stats. */ -int run_tests(void) { - GREATEST_INIT(); /* init. greatest internals */ - /* List of suites to run (if any). */ - RUN_SUITE(suite); - /* Tests can also be run directly, without using test suites. */ - RUN_TEST(foo_should_foo); - GREATEST_PRINT_REPORT(); /* display results */ - return greatest_all_passed(); -} -/* main(), for a standalone command-line test runner. - * This replaces run_tests above, and adds command line option - * handling and exiting with a pass/fail status. */ -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* init & parse command-line args */ - RUN_SUITE(suite); - GREATEST_MAIN_END(); /* display results */ -} -#endif -/*********************************************************************/ - - -#include -#include -#include -#include - -/*********** - * Options * - ***********/ - -/* Default column width for non-verbose output. */ -#ifndef GREATEST_DEFAULT_WIDTH -#define GREATEST_DEFAULT_WIDTH 72 -#endif - -/* FILE *, for test logging. */ -#ifndef GREATEST_STDOUT -#define GREATEST_STDOUT stdout -#endif - -/* Remove GREATEST_ prefix from most commonly used symbols? */ -#ifndef GREATEST_USE_ABBREVS -#define GREATEST_USE_ABBREVS 1 -#endif - -/* Set to 0 to disable all use of setjmp/longjmp. */ -#ifndef GREATEST_USE_LONGJMP -#define GREATEST_USE_LONGJMP 1 -#endif - -#if GREATEST_USE_LONGJMP -#include -#endif - -/* Set to 0 to disable all use of time.h / clock(). */ -#ifndef GREATEST_USE_TIME -#define GREATEST_USE_TIME 1 -#endif - -#if GREATEST_USE_TIME -#include -#endif - -/* Floating point type, for ASSERT_IN_RANGE. */ -#ifndef GREATEST_FLOAT -#define GREATEST_FLOAT double -#define GREATEST_FLOAT_FMT "%g" -#endif - -/********* - * Types * - *********/ - -/* Info for the current running suite. */ -typedef struct greatest_suite_info { - unsigned int tests_run; - unsigned int passed; - unsigned int failed; - unsigned int skipped; - -#if GREATEST_USE_TIME - /* timers, pre/post running suite and individual tests */ - clock_t pre_suite; - clock_t post_suite; - clock_t pre_test; - clock_t post_test; -#endif -} greatest_suite_info; - -/* Type for a suite function. */ -typedef void (greatest_suite_cb)(void); - -/* Types for setup/teardown callbacks. If non-NULL, these will be run - * and passed the pointer to their additional data. */ -typedef void (greatest_setup_cb)(void *udata); -typedef void (greatest_teardown_cb)(void *udata); - -/* Type for an equality comparison between two pointers of the same type. - * Should return non-0 if equal, otherwise 0. - * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ -typedef int greatest_equal_cb(const void *exp, const void *got, void *udata); - -/* Type for a callback that prints a value pointed to by T. - * Return value has the same meaning as printf's. - * UDATA is a closure value, passed through from ASSERT_EQUAL_T[m]. */ -typedef int greatest_printf_cb(const void *t, void *udata); - -/* Callbacks for an arbitrary type; needed for type-specific - * comparisons via GREATEST_ASSERT_EQUAL_T[m].*/ -typedef struct greatest_type_info { - greatest_equal_cb *equal; - greatest_printf_cb *print; -} greatest_type_info; - -typedef struct greatest_memory_cmp_env { - const unsigned char *exp; - const unsigned char *got; - size_t size; -} greatest_memory_cmp_env; - -/* Callbacks for string and raw memory types. */ -extern greatest_type_info greatest_type_info_string; -extern greatest_type_info greatest_type_info_memory; - -typedef enum { - GREATEST_FLAG_FIRST_FAIL = 0x01, - GREATEST_FLAG_LIST_ONLY = 0x02 -} greatest_flag_t; - -/* Struct containing all test runner state. */ -typedef struct greatest_run_info { - unsigned char flags; - unsigned char verbosity; - unsigned int tests_run; /* total test count */ - - /* overall pass/fail/skip counts */ - unsigned int passed; - unsigned int failed; - unsigned int skipped; - unsigned int assertions; - - /* currently running test suite */ - greatest_suite_info suite; - - /* info to print about the most recent failure */ - const char *fail_file; - unsigned int fail_line; - const char *msg; - - /* current setup/teardown hooks and userdata */ - greatest_setup_cb *setup; - void *setup_udata; - greatest_teardown_cb *teardown; - void *teardown_udata; - - /* formatting info for ".....s...F"-style output */ - unsigned int col; - unsigned int width; - - /* only run a specific suite or test */ - const char *suite_filter; - const char *test_filter; - -#if GREATEST_USE_TIME - /* overall timers */ - clock_t begin; - clock_t end; -#endif - -#if GREATEST_USE_LONGJMP - jmp_buf jump_dest; -#endif -} greatest_run_info; - -struct greatest_report_t { - /* overall pass/fail/skip counts */ - unsigned int passed; - unsigned int failed; - unsigned int skipped; - unsigned int assertions; -}; - -/* Global var for the current testing context. - * Initialized by GREATEST_MAIN_DEFS(). */ -extern greatest_run_info greatest_info; - -/* Type for ASSERT_ENUM_EQ's ENUM_STR argument. */ -typedef const char *greatest_enum_str_fun(int value); - -/********************** - * Exported functions * - **********************/ - -/* These are used internally by greatest. */ -void greatest_do_pass(const char *name); -void greatest_do_fail(const char *name); -void greatest_do_skip(const char *name); -int greatest_pre_test(const char *name); -void greatest_post_test(const char *name, int res); -void greatest_usage(const char *name); -int greatest_do_assert_equal_t(const void *exp, const void *got, - greatest_type_info *type_info, void *udata); - -/* These are part of the public greatest API. */ -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); -int greatest_all_passed(void); -void greatest_set_test_filter(const char *name); -void greatest_set_suite_filter(const char *name); -void greatest_get_report(struct greatest_report_t *report); -unsigned int greatest_get_verbosity(void); -void greatest_set_verbosity(unsigned int verbosity); -void greatest_set_flag(greatest_flag_t flag); - - -/******************** -* Language Support * -********************/ - -/* If __VA_ARGS__ (C99) is supported, allow parametric testing -* without needing to manually manage the argument struct. */ -#if __STDC_VERSION__ >= 19901L || _MSC_VER >= 1800 -#define GREATEST_VA_ARGS -#endif - - -/********** - * Macros * - **********/ - -/* Define a suite. */ -#define GREATEST_SUITE(NAME) void NAME(void); void NAME(void) - -/* Declare a suite, provided by another compilation unit. */ -#define GREATEST_SUITE_EXTERN(NAME) void NAME(void) - -/* Start defining a test function. - * The arguments are not included, to allow parametric testing. */ -#define GREATEST_TEST static enum greatest_test_res - -/* PASS/FAIL/SKIP result from a test. Used internally. */ -typedef enum greatest_test_res { - GREATEST_TEST_RES_PASS = 0, - GREATEST_TEST_RES_FAIL = -1, - GREATEST_TEST_RES_SKIP = 1 -} greatest_test_res; - -/* Run a suite. */ -#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) - -/* Run a test in the current suite. */ -#define GREATEST_RUN_TEST(TEST) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - enum greatest_test_res res = GREATEST_SAVE_CONTEXT(); \ - if (res == GREATEST_TEST_RES_PASS) { \ - res = TEST(); \ - } \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -/* Ignore a test, don't warn about it being unused. */ -#define GREATEST_IGNORE_TEST(TEST) (void)TEST - -/* Run a test in the current suite with one void * argument, - * which can be a pointer to a struct with multiple arguments. */ -#define GREATEST_RUN_TEST1(TEST, ENV) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(ENV); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -#ifdef GREATEST_VA_ARGS -#define GREATEST_RUN_TESTp(TEST, ...) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(__VA_ARGS__); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) -#endif - - -/* Check if the test runner is in verbose mode. */ -#define GREATEST_IS_VERBOSE() ((greatest_info.verbosity) > 0) -#define GREATEST_LIST_ONLY() \ - (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) -#define GREATEST_FIRST_FAIL() \ - (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) -#define GREATEST_FAILURE_ABORT() \ - (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) - -/* Message-less forms of tests defined below. */ -#define GREATEST_PASS() GREATEST_PASSm(NULL) -#define GREATEST_FAIL() GREATEST_FAILm(NULL) -#define GREATEST_SKIP() GREATEST_SKIPm(NULL) -#define GREATEST_ASSERT(COND) \ - GREATEST_ASSERTm(#COND, COND) -#define GREATEST_ASSERT_OR_LONGJMP(COND) \ - GREATEST_ASSERT_OR_LONGJMPm(#COND, COND) -#define GREATEST_ASSERT_FALSE(COND) \ - GREATEST_ASSERT_FALSEm(#COND, COND) -#define GREATEST_ASSERT_EQ(EXP, GOT) \ - GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) -#define GREATEST_ASSERT_EQ_FMT(EXP, GOT, FMT) \ - GREATEST_ASSERT_EQ_FMTm(#EXP " != " #GOT, EXP, GOT, FMT) -#define GREATEST_ASSERT_IN_RANGE(EXP, GOT, TOL) \ - GREATEST_ASSERT_IN_RANGEm(#EXP " != " #GOT " +/- " #TOL, EXP, GOT, TOL) -#define GREATEST_ASSERT_EQUAL_T(EXP, GOT, TYPE_INFO, UDATA) \ - GREATEST_ASSERT_EQUAL_Tm(#EXP " != " #GOT, EXP, GOT, TYPE_INFO, UDATA) -#define GREATEST_ASSERT_STR_EQ(EXP, GOT) \ - GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) -#define GREATEST_ASSERT_STRN_EQ(EXP, GOT, SIZE) \ - GREATEST_ASSERT_STRN_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) -#define GREATEST_ASSERT_MEM_EQ(EXP, GOT, SIZE) \ - GREATEST_ASSERT_MEM_EQm(#EXP " != " #GOT, EXP, GOT, SIZE) -#define GREATEST_ASSERT_ENUM_EQ(EXP, GOT, ENUM_STR) \ - GREATEST_ASSERT_ENUM_EQm(#EXP " != " #GOT, EXP, GOT, ENUM_STR) - -/* The following forms take an additional message argument first, - * to be displayed by the test runner. */ - -/* Fail if a condition is not true, with message. */ -#define GREATEST_ASSERTm(MSG, COND) \ - do { \ - greatest_info.assertions++; \ - if (!(COND)) { GREATEST_FAILm(MSG); } \ - } while (0) - -/* Fail if a condition is not true, longjmping out of test. */ -#define GREATEST_ASSERT_OR_LONGJMPm(MSG, COND) \ - do { \ - greatest_info.assertions++; \ - if (!(COND)) { GREATEST_FAIL_WITH_LONGJMPm(MSG); } \ - } while (0) - -/* Fail if a condition is not false, with message. */ -#define GREATEST_ASSERT_FALSEm(MSG, COND) \ - do { \ - greatest_info.assertions++; \ - if ((COND)) { GREATEST_FAILm(MSG); } \ - } while (0) - -/* Fail if EXP != GOT (equality comparison by ==). */ -#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ - do { \ - greatest_info.assertions++; \ - if ((EXP) != (GOT)) { GREATEST_FAILm(MSG); } \ - } while (0) - -/* Fail if EXP != GOT (equality comparison by ==). - * Warning: EXP and GOT will be evaluated more than once on failure. */ -#define GREATEST_ASSERT_EQ_FMTm(MSG, EXP, GOT, FMT) \ - do { \ - const char *greatest_FMT = ( FMT ); \ - greatest_info.assertions++; \ - if ((EXP) != (GOT)) { \ - fprintf(GREATEST_STDOUT, "\nExpected: "); \ - fprintf(GREATEST_STDOUT, greatest_FMT, EXP); \ - fprintf(GREATEST_STDOUT, "\n Got: "); \ - fprintf(GREATEST_STDOUT, greatest_FMT, GOT); \ - fprintf(GREATEST_STDOUT, "\n"); \ - GREATEST_FAILm(MSG); \ - } \ - } while (0) - -/* Fail if EXP is not equal to GOT, printing enum IDs. */ -#define GREATEST_ASSERT_ENUM_EQm(MSG, EXP, GOT, ENUM_STR) \ - do { \ - int greatest_EXP = (int)(EXP); \ - int greatest_GOT = (int)(GOT); \ - greatest_enum_str_fun *greatest_ENUM_STR = ENUM_STR; \ - if (greatest_EXP != greatest_GOT) { \ - fprintf(GREATEST_STDOUT, "\nExpected: %s", \ - greatest_ENUM_STR(greatest_EXP)); \ - fprintf(GREATEST_STDOUT, "\n Got: %s\n", \ - greatest_ENUM_STR(greatest_GOT)); \ - GREATEST_FAILm(MSG); \ - } \ - } while (0) \ - -/* Fail if GOT not in range of EXP +|- TOL. */ -#define GREATEST_ASSERT_IN_RANGEm(MSG, EXP, GOT, TOL) \ - do { \ - GREATEST_FLOAT greatest_EXP = (EXP); \ - GREATEST_FLOAT greatest_GOT = (GOT); \ - GREATEST_FLOAT greatest_TOL = (TOL); \ - greatest_info.assertions++; \ - if ((greatest_EXP > greatest_GOT && \ - greatest_EXP - greatest_GOT > greatest_TOL) || \ - (greatest_EXP < greatest_GOT && \ - greatest_GOT - greatest_EXP > greatest_TOL)) { \ - fprintf(GREATEST_STDOUT, \ - "\nExpected: " GREATEST_FLOAT_FMT \ - " +/- " GREATEST_FLOAT_FMT \ - "\n Got: " GREATEST_FLOAT_FMT \ - "\n", \ - greatest_EXP, greatest_TOL, greatest_GOT); \ - GREATEST_FAILm(MSG); \ - } \ - } while (0) - -/* Fail if EXP is not equal to GOT, according to strcmp. */ -#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ - do { \ - GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ - &greatest_type_info_string, NULL); \ - } while (0) \ - -/* Fail if EXP is not equal to GOT, according to strcmp. */ -#define GREATEST_ASSERT_STRN_EQm(MSG, EXP, GOT, SIZE) \ - do { \ - size_t size = SIZE; \ - GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, \ - &greatest_type_info_string, &size); \ - } while (0) \ - -/* Fail if EXP is not equal to GOT, according to memcmp. */ -#define GREATEST_ASSERT_MEM_EQm(MSG, EXP, GOT, SIZE) \ - do { \ - greatest_memory_cmp_env env; \ - env.exp = (const unsigned char *)EXP; \ - env.got = (const unsigned char *)GOT; \ - env.size = SIZE; \ - GREATEST_ASSERT_EQUAL_Tm(MSG, env.exp, env.got, \ - &greatest_type_info_memory, &env); \ - } while (0) \ - -/* Fail if EXP is not equal to GOT, according to a comparison - * callback in TYPE_INFO. If they are not equal, optionally use a - * print callback in TYPE_INFO to print them. */ -#define GREATEST_ASSERT_EQUAL_Tm(MSG, EXP, GOT, TYPE_INFO, UDATA) \ - do { \ - greatest_type_info *type_info = (TYPE_INFO); \ - greatest_info.assertions++; \ - if (!greatest_do_assert_equal_t(EXP, GOT, \ - type_info, UDATA)) { \ - if (type_info == NULL || type_info->equal == NULL) { \ - GREATEST_FAILm("type_info->equal callback missing!"); \ - } else { \ - GREATEST_FAILm(MSG); \ - } \ - } \ - } while (0) \ - -/* Pass. */ -#define GREATEST_PASSm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return GREATEST_TEST_RES_PASS; \ - } while (0) - -/* Fail. */ -#define GREATEST_FAILm(MSG) \ - do { \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - greatest_info.msg = MSG; \ - return GREATEST_TEST_RES_FAIL; \ - } while (0) - -/* Optional GREATEST_FAILm variant that longjmps. */ -#if GREATEST_USE_LONGJMP -#define GREATEST_FAIL_WITH_LONGJMP() GREATEST_FAIL_WITH_LONGJMPm(NULL) -#define GREATEST_FAIL_WITH_LONGJMPm(MSG) \ - do { \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - greatest_info.msg = MSG; \ - longjmp(greatest_info.jump_dest, GREATEST_TEST_RES_FAIL); \ - } while (0) -#endif - -/* Skip the current test. */ -#define GREATEST_SKIPm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return GREATEST_TEST_RES_SKIP; \ - } while (0) - -/* Check the result of a subfunction using ASSERT, etc. */ -#define GREATEST_CHECK_CALL(RES) \ - do { \ - enum greatest_test_res greatest_RES = RES; \ - if (greatest_RES != GREATEST_TEST_RES_PASS) { \ - return greatest_RES; \ - } \ - } while (0) \ - -#if GREATEST_USE_TIME -#define GREATEST_SET_TIME(NAME) \ - NAME = clock(); \ - if (NAME == (clock_t) -1) { \ - fprintf(GREATEST_STDOUT, \ - "clock error: %s\n", #NAME); \ - exit(EXIT_FAILURE); \ - } - -#define GREATEST_CLOCK_DIFF(C1, C2) \ - fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ - (long unsigned int) (C2) - (long unsigned int)(C1), \ - (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) -#else -#define GREATEST_SET_TIME(UNUSED) -#define GREATEST_CLOCK_DIFF(UNUSED1, UNUSED2) -#endif - -#if GREATEST_USE_LONGJMP -#define GREATEST_SAVE_CONTEXT() \ - /* setjmp returns 0 (GREATEST_TEST_RES_PASS) on first call */ \ - /* so the test runs, then RES_FAIL from FAIL_WITH_LONGJMP. */ \ - ((enum greatest_test_res)(setjmp(greatest_info.jump_dest))) -#else -#define GREATEST_SAVE_CONTEXT() \ - /*a no-op, since setjmp/longjmp aren't being used */ \ - GREATEST_TEST_RES_PASS -#endif - -/* Include several function definitions in the main test file. */ -#define GREATEST_MAIN_DEFS() \ - \ -/* Is FILTER a subset of NAME? */ \ -static int greatest_name_match(const char *name, \ - const char *filter) { \ - size_t offset = 0; \ - size_t filter_len = strlen(filter); \ - while (name[offset] != '\0') { \ - if (name[offset] == filter[0]) { \ - if (0 == strncmp(&name[offset], filter, filter_len)) { \ - return 1; \ - } \ - } \ - offset++; \ - } \ - \ - return 0; \ -} \ - \ -int greatest_pre_test(const char *name) { \ - if (!GREATEST_LIST_ONLY() \ - && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ - && (greatest_info.test_filter == NULL || \ - greatest_name_match(name, greatest_info.test_filter))) { \ - GREATEST_SET_TIME(greatest_info.suite.pre_test); \ - if (greatest_info.setup) { \ - greatest_info.setup(greatest_info.setup_udata); \ - } \ - return 1; /* test should be run */ \ - } else { \ - return 0; /* skipped */ \ - } \ -} \ - \ -void greatest_post_test(const char *name, int res) { \ - GREATEST_SET_TIME(greatest_info.suite.post_test); \ - if (greatest_info.teardown) { \ - void *udata = greatest_info.teardown_udata; \ - greatest_info.teardown(udata); \ - } \ - \ - if (res <= GREATEST_TEST_RES_FAIL) { \ - greatest_do_fail(name); \ - } else if (res >= GREATEST_TEST_RES_SKIP) { \ - greatest_do_skip(name); \ - } else if (res == GREATEST_TEST_RES_PASS) { \ - greatest_do_pass(name); \ - } \ - greatest_info.suite.tests_run++; \ - greatest_info.col++; \ - if (GREATEST_IS_VERBOSE()) { \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ - greatest_info.suite.post_test); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } else if (greatest_info.col % greatest_info.width == 0) { \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - } \ - if (GREATEST_STDOUT == stdout) fflush(stdout); \ -} \ - \ -static void report_suite(void) { \ - if (greatest_info.suite.tests_run > 0) { \ - fprintf(GREATEST_STDOUT, \ - "\n%u test%s - %u passed, %u failed, %u skipped", \ - greatest_info.suite.tests_run, \ - greatest_info.suite.tests_run == 1 ? "" : "s", \ - greatest_info.suite.passed, \ - greatest_info.suite.failed, \ - greatest_info.suite.skipped); \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ - greatest_info.suite.post_suite); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } \ -} \ - \ -static void update_counts_and_reset_suite(void) { \ - greatest_info.setup = NULL; \ - greatest_info.setup_udata = NULL; \ - greatest_info.teardown = NULL; \ - greatest_info.teardown_udata = NULL; \ - greatest_info.passed += greatest_info.suite.passed; \ - greatest_info.failed += greatest_info.suite.failed; \ - greatest_info.skipped += greatest_info.suite.skipped; \ - greatest_info.tests_run += greatest_info.suite.tests_run; \ - memset(&greatest_info.suite, 0, sizeof(greatest_info.suite)); \ - greatest_info.col = 0; \ -} \ - \ -static void greatest_run_suite(greatest_suite_cb *suite_cb, \ - const char *suite_name) { \ - if (greatest_info.suite_filter && \ - !greatest_name_match(suite_name, greatest_info.suite_filter)) { \ - return; \ - } \ - update_counts_and_reset_suite(); \ - if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) { return; } \ - fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ - GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ - suite_cb(); \ - GREATEST_SET_TIME(greatest_info.suite.post_suite); \ - report_suite(); \ -} \ - \ -void greatest_do_pass(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "PASS %s: %s", \ - name, greatest_info.msg ? greatest_info.msg : ""); \ - } else { \ - fprintf(GREATEST_STDOUT, "."); \ - } \ - greatest_info.suite.passed++; \ -} \ - \ -void greatest_do_fail(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, \ - "FAIL %s: %s (%s:%u)", \ - name, greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } else { \ - fprintf(GREATEST_STDOUT, "F"); \ - greatest_info.col++; \ - /* add linebreak if in line of '.'s */ \ - if (greatest_info.col != 0) { \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - } \ - fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ - name, \ - greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } \ - greatest_info.suite.failed++; \ -} \ - \ -void greatest_do_skip(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ - name, \ - greatest_info.msg ? \ - greatest_info.msg : "" ); \ - } else { \ - fprintf(GREATEST_STDOUT, "s"); \ - } \ - greatest_info.suite.skipped++; \ -} \ - \ -int greatest_do_assert_equal_t(const void *exp, const void *got, \ - greatest_type_info *type_info, void *udata) { \ - int eq = 0; \ - if (type_info == NULL || type_info->equal == NULL) { \ - return 0; \ - } \ - eq = type_info->equal(exp, got, udata); \ - if (!eq) { \ - if (type_info->print != NULL) { \ - fprintf(GREATEST_STDOUT, "\nExpected: "); \ - (void)type_info->print(exp, udata); \ - fprintf(GREATEST_STDOUT, "\n Got: "); \ - (void)type_info->print(got, udata); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } else { \ - fprintf(GREATEST_STDOUT, \ - "GREATEST_ASSERT_EQUAL_T failure at %s:%u\n", \ - greatest_info.fail_file, \ - greatest_info.fail_line); \ - } \ - } \ - return eq; \ -} \ - \ -void greatest_usage(const char *name) { \ - fprintf(GREATEST_STDOUT, \ - "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ - " -h, --help print this Help\n" \ - " -l List suites and their tests, then exit\n" \ - " -f Stop runner after first failure\n" \ - " -v Verbose output\n" \ - " -s SUITE only run suites containing string SUITE\n" \ - " -t TEST only run tests containing string TEST\n", \ - name); \ -} \ - \ -static void greatest_parse_args(int argc, char **argv) { \ - int i = 0; \ - for (i = 1; i < argc; i++) { \ - if (0 == strncmp("-t", argv[i], 2)) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.test_filter = argv[i+1]; \ - i++; \ - } else if (0 == strncmp("-s", argv[i], 2)) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.suite_filter = argv[i+1]; \ - i++; \ - } else if (0 == strncmp("-f", argv[i], 2)) { \ - greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ - } else if (0 == strncmp("-v", argv[i], 2)) { \ - greatest_info.verbosity++; \ - } else if (0 == strncmp("-l", argv[i], 2)) { \ - greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ - } else if (0 == strncmp("-h", argv[i], 2) || \ - 0 == strncmp("--help", argv[i], 6)) { \ - greatest_usage(argv[0]); \ - exit(EXIT_SUCCESS); \ - } else if (0 == strncmp("--", argv[i], 2)) { \ - break; \ - } else { \ - fprintf(GREATEST_STDOUT, \ - "Unknown argument '%s'\n", argv[i]); \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - } \ -} \ - \ -int greatest_all_passed(void) { return (greatest_info.failed == 0); } \ - \ -void greatest_set_test_filter(const char *name) { \ - greatest_info.test_filter = name; \ -} \ - \ -void greatest_set_suite_filter(const char *name) { \ - greatest_info.suite_filter = name; \ -} \ - \ -void greatest_get_report(struct greatest_report_t *report) { \ - if (report) { \ - report->passed = greatest_info.passed; \ - report->failed = greatest_info.failed; \ - report->skipped = greatest_info.skipped; \ - report->assertions = greatest_info.assertions; \ - } \ -} \ - \ -unsigned int greatest_get_verbosity(void) { \ - return greatest_info.verbosity; \ -} \ - \ -void greatest_set_verbosity(unsigned int verbosity) { \ - greatest_info.verbosity = (unsigned char)verbosity; \ -} \ - \ -void greatest_set_flag(greatest_flag_t flag) { \ - greatest_info.flags |= flag; \ -} \ - \ -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ - greatest_info.setup = cb; \ - greatest_info.setup_udata = udata; \ -} \ - \ -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ - void *udata) { \ - greatest_info.teardown = cb; \ - greatest_info.teardown_udata = udata; \ -} \ - \ -static int greatest_string_equal_cb(const void *exp, const void *got, \ - void *udata) { \ - size_t *size = (size_t *)udata; \ - return (size != NULL \ - ? (0 == strncmp((const char *)exp, (const char *)got, *size)) \ - : (0 == strcmp((const char *)exp, (const char *)got))); \ -} \ - \ -static int greatest_string_printf_cb(const void *t, void *udata) { \ - (void)udata; /* note: does not check \0 termination. */ \ - return fprintf(GREATEST_STDOUT, "%s", (const char *)t); \ -} \ - \ -greatest_type_info greatest_type_info_string = { \ - greatest_string_equal_cb, \ - greatest_string_printf_cb, \ -}; \ - \ -static int greatest_memory_equal_cb(const void *exp, const void *got, \ - void *udata) { \ - greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ - return (0 == memcmp(exp, got, env->size)); \ -} \ - \ -static int greatest_memory_printf_cb(const void *t, void *udata) { \ - greatest_memory_cmp_env *env = (greatest_memory_cmp_env *)udata; \ - unsigned char *buf = (unsigned char *)t, diff_mark = ' '; \ - FILE *out = GREATEST_STDOUT; \ - size_t i, line_i, line_len = 0; \ - int len = 0; /* format hexdump with differences highlighted */ \ - for (i = 0; i < env->size; i+= line_len) { \ - diff_mark = ' '; \ - line_len = env->size - i; \ - if (line_len > 16) { line_len = 16; } \ - for (line_i = i; line_i < i + line_len; line_i++) { \ - if (env->exp[line_i] != env->got[line_i]) diff_mark = 'X'; \ - } \ - len += fprintf(out, "\n%04x %c ", (unsigned int)i, diff_mark); \ - for (line_i = i; line_i < i + line_len; line_i++) { \ - int m = env->exp[line_i] == env->got[line_i]; /* match? */ \ - len += fprintf(out, "%02x%c", buf[line_i], m ? ' ' : '<'); \ - } \ - for (line_i = 0; line_i < 16 - line_len; line_i++) { \ - len += fprintf(out, " "); \ - } \ - fprintf(out, " "); \ - for (line_i = i; line_i < i + line_len; line_i++) { \ - unsigned char c = buf[line_i]; \ - len += fprintf(out, "%c", isprint(c) ? c : '.'); \ - } \ - } \ - len += fprintf(out, "\n"); \ - return len; \ -} \ - \ -greatest_type_info greatest_type_info_memory = { \ - greatest_memory_equal_cb, \ - greatest_memory_printf_cb, \ -}; \ - \ -greatest_run_info greatest_info - -/* Init internals. */ -#define GREATEST_INIT() \ - do { \ - /* Suppress unused function warning if features aren't used */ \ - (void)greatest_run_suite; \ - (void)greatest_parse_args; \ - \ - memset(&greatest_info, 0, sizeof(greatest_info)); \ - greatest_info.width = GREATEST_DEFAULT_WIDTH; \ - GREATEST_SET_TIME(greatest_info.begin); \ - } while (0) \ - -/* Handle command-line arguments, etc. */ -#define GREATEST_MAIN_BEGIN() \ - do { \ - GREATEST_INIT(); \ - greatest_parse_args(argc, argv); \ - } while (0) - -/* Report passes, failures, skipped tests, the number of - * assertions, and the overall run time. */ -#define GREATEST_PRINT_REPORT() \ - do { \ - if (!GREATEST_LIST_ONLY()) { \ - update_counts_and_reset_suite(); \ - GREATEST_SET_TIME(greatest_info.end); \ - fprintf(GREATEST_STDOUT, \ - "\nTotal: %u test%s", \ - greatest_info.tests_run, \ - greatest_info.tests_run == 1 ? "" : "s"); \ - GREATEST_CLOCK_DIFF(greatest_info.begin, \ - greatest_info.end); \ - fprintf(GREATEST_STDOUT, ", %u assertion%s\n", \ - greatest_info.assertions, \ - greatest_info.assertions == 1 ? "" : "s"); \ - fprintf(GREATEST_STDOUT, \ - "Pass: %u, fail: %u, skip: %u.\n", \ - greatest_info.passed, \ - greatest_info.failed, greatest_info.skipped); \ - } \ - } while (0) - -/* Report results, exit with exit status based on results. */ -#define GREATEST_MAIN_END() \ - do { \ - GREATEST_PRINT_REPORT(); \ - return (greatest_all_passed() ? EXIT_SUCCESS : EXIT_FAILURE); \ - } while (0) - -/* Make abbreviations without the GREATEST_ prefix for the - * most commonly used symbols. */ -#if GREATEST_USE_ABBREVS -#define TEST GREATEST_TEST -#define SUITE GREATEST_SUITE -#define SUITE_EXTERN GREATEST_SUITE_EXTERN -#define RUN_TEST GREATEST_RUN_TEST -#define RUN_TEST1 GREATEST_RUN_TEST1 -#define RUN_SUITE GREATEST_RUN_SUITE -#define IGNORE_TEST GREATEST_IGNORE_TEST -#define ASSERT GREATEST_ASSERT -#define ASSERTm GREATEST_ASSERTm -#define ASSERT_FALSE GREATEST_ASSERT_FALSE -#define ASSERT_EQ GREATEST_ASSERT_EQ -#define ASSERT_EQ_FMT GREATEST_ASSERT_EQ_FMT -#define ASSERT_IN_RANGE GREATEST_ASSERT_IN_RANGE -#define ASSERT_EQUAL_T GREATEST_ASSERT_EQUAL_T -#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ -#define ASSERT_STRN_EQ GREATEST_ASSERT_STRN_EQ -#define ASSERT_MEM_EQ GREATEST_ASSERT_MEM_EQ -#define ASSERT_ENUM_EQ GREATEST_ASSERT_ENUM_EQ -#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm -#define ASSERT_EQm GREATEST_ASSERT_EQm -#define ASSERT_EQ_FMTm GREATEST_ASSERT_EQ_FMTm -#define ASSERT_IN_RANGEm GREATEST_ASSERT_IN_RANGEm -#define ASSERT_EQUAL_Tm GREATEST_ASSERT_EQUAL_Tm -#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm -#define ASSERT_STRN_EQm GREATEST_ASSERT_STRN_EQm -#define ASSERT_MEM_EQm GREATEST_ASSERT_MEM_EQm -#define ASSERT_ENUM_EQm GREATEST_ASSERT_ENUM_EQm -#define PASS GREATEST_PASS -#define FAIL GREATEST_FAIL -#define SKIP GREATEST_SKIP -#define PASSm GREATEST_PASSm -#define FAILm GREATEST_FAILm -#define SKIPm GREATEST_SKIPm -#define SET_SETUP GREATEST_SET_SETUP_CB -#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB -#define CHECK_CALL GREATEST_CHECK_CALL - -#ifdef GREATEST_VA_ARGS -#define RUN_TESTp GREATEST_RUN_TESTp -#endif - -#if GREATEST_USE_LONGJMP -#define ASSERT_OR_LONGJMP GREATEST_ASSERT_OR_LONGJMP -#define ASSERT_OR_LONGJMPm GREATEST_ASSERT_OR_LONGJMPm -#define FAIL_WITH_LONGJMP GREATEST_FAIL_WITH_LONGJMP -#define FAIL_WITH_LONGJMPm GREATEST_FAIL_WITH_LONGJMPm -#endif - -#endif /* USE_ABBREVS */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/common/thirdparty/patches/.gitattributes b/src/common/thirdparty/patches/.gitattributes deleted file mode 100644 index 9812ceb1f..000000000 --- a/src/common/thirdparty/patches/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.patch text eol=lf diff --git a/src/common/thirdparty/patches/windows/python-pyconfig.patch b/src/common/thirdparty/patches/windows/python-pyconfig.patch deleted file mode 100644 index 4280dee77..000000000 --- a/src/common/thirdparty/patches/windows/python-pyconfig.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff --git a/inc/Windows/pyconfig.h b/inc/Windows/pyconfig.h -index 1cfc59b..d4861cb ---- a/inc/Windows/pyconfig.h -+++ b/inc/Windows/pyconfig.h -@@ -1,6 +1,11 @@ - #ifndef Py_CONFIG_H - #define Py_CONFIG_H - -+#ifdef _MSC_VER -+#pragma push_macro("_DEBUG") -+#undef _DEBUG -+#endif -+ - /* pyconfig.h. NOT Generated automatically by configure. - - This is a manually maintained version used for the Watcom, -@@ -756,4 +761,8 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ - least significant byte first */ - #define DOUBLE_IS_LITTLE_ENDIAN_IEEE754 1 - -+#ifdef _MSC_VER -+#pragma pop_macro("_DEBUG") -+#endif -+ - #endif /* !Py_CONFIG_H */ diff --git a/src/common/thirdparty/patches/windows/redis.patch b/src/common/thirdparty/patches/windows/redis.patch deleted file mode 100644 index 5ed2df510..000000000 --- a/src/common/thirdparty/patches/windows/redis.patch +++ /dev/null @@ -1,772 +0,0 @@ -diff --git a/msvs/RedisServer.vcxproj b/msvs/RedisServer.vcxproj -index 115ce90..68afb44 ---- a/msvs/RedisServer.vcxproj -+++ b/msvs/RedisServer.vcxproj -@@ -24,26 +24,26 @@ - - - -- Application -+ StaticLibrary - true -- v120 -+ v140_xp - false - - -- Application -+ StaticLibrary - true -- v120 -+ v140_xp - false - - -- Application -+ StaticLibrary - false -- v120 -+ v140_xp - - -- Application -+ StaticLibrary - false -- v120 -+ v140_xp - - - -@@ -61,41 +61,23 @@ - - - -- -+ - false - redis-server - false -- -- -- false -- redis-server -- false -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -- -- -- false -- redis-server -- false -- Build -- -- -- false -- redis-server -- false -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\ - - - -- USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions) -- $(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include -- MultiThreadedDebug -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions) -+ $(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include - Level3 - ProgramDatabase - Disabled - 4996;4146 -- true -+ false -+ true - - - true -@@ -109,14 +91,14 @@ - - - -- USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501 -- $(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include -- MultiThreadedDebug -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;_DEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501 -+ $(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include - Level3 - ProgramDatabase - Disabled - 4996;4146 -- true -+ false -+ true - - - true -@@ -130,14 +112,13 @@ - - - -- USE_JEMALLOC;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions) -- $(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include -- MultiThreaded -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions) -+ $(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include - Level3 - ProgramDatabase - 4996;4146 -- true - Full -+ true - - - true -@@ -162,13 +143,12 @@ - - - -- USE_JEMALLOC;_OFF_T_DEFINED;_WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501 -- $(SolutionDir)..\deps\lua\src;$(SolutionDir)..\deps\hiredis;$(SolutionDir)..\deps\jemalloc-win\include -- MultiThreaded -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;_WIN32;LACKS_STDLIB_H;NDEBUG;_CONSOLE;__x86_64__;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501 -+ $(ProjectDir)..\deps\lua\src;$(ProjectDir)..\deps\hiredis;$(ProjectDir)..\deps\jemalloc-win\include - Level3 - ProgramDatabase - 4996;4146 -- true -+ true - - - true -@@ -271,9 +251,6 @@ - - - -- -- {8b897e33-6428-4254-8335-4911d179bad1} -- - - {8c07f811-c81c-432c-b334-1ae6faecf951} - -diff --git a/msvs/hiredis/hiredis.vcxproj b/msvs/hiredis/hiredis.vcxproj -index 0622958..efaedae ---- a/msvs/hiredis/hiredis.vcxproj -+++ b/msvs/hiredis/hiredis.vcxproj -@@ -28,27 +28,25 @@ - StaticLibrary - true - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - true - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - false -- true - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - false -- true - MultiByte -- v120 -+ v140_xp - - - -@@ -66,30 +64,20 @@ - - - -- -+ - hiredis -- -- -- hiredis -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -- -- -- hiredis -- -- -- hiredis -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -+ $(ProjectDir)..\$(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\ - - - - NotUsing - Level3 - Disabled -- _OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions) -- MultiThreadedDebug -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions) - 4996 -+ false -+ true - - - Windows -@@ -101,9 +89,10 @@ - NotUsing - Level3 - Disabled -- _OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions) -- MultiThreadedDebug -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;_DEBUG;%(PreprocessorDefinitions) - 4996 -+ false -+ true - - - Windows -@@ -117,10 +106,9 @@ - Full - true - true -- _OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions) -- MultiThreaded -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions) - 4996 -- true -+ true - - - Windows -@@ -136,10 +124,9 @@ - Full - true - true -- _OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions) -- MultiThreaded -+ _WIN32_WINNT=0x0502;_OFF_T_DEFINED;WIN32;_LIB;%(PreprocessorDefinitions) - 4996 -- true -+ true - - - Windows -diff --git a/msvs/lua/lua/lua.vcxproj b/msvs/lua/lua/lua.vcxproj -index b187130..adef07b ---- a/msvs/lua/lua/lua.vcxproj -+++ b/msvs/lua/lua/lua.vcxproj -@@ -30,28 +30,28 @@ - true - false - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - true - false - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - false - false - MultiByte -- v120 -+ v140_xp - - - StaticLibrary - false - false - MultiByte -- v120 -+ v140_xp - - - -@@ -69,25 +69,16 @@ - - - -- -+ - true -- .lib -- -- -- true -- .lib -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ - -- -+ - false -- .lib - -- -- false -+ - .lib -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\ - - - -@@ -95,8 +86,9 @@ - Disabled - _OFF_T_DEFINED;WIN32;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);LUA_ANSI;ENABLE_CJSON_GLOBAL - NotUsing -- MultiThreadedDebug - 4244;4018 -+ false -+ true - - - true -@@ -110,8 +102,9 @@ - Disabled - _OFF_T_DEFINED;WIN32;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501;LUA_ANSI;ENABLE_CJSON_GLOBAL - NotUsing -- MultiThreadedDebug - 4244;4018 -+ false -+ true - - - true -@@ -124,10 +117,10 @@ - Level3 - _OFF_T_DEFINED;WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);LUA_ANSI;ENABLE_CJSON_GLOBAL - NotUsing -- MultiThreaded - 4244;4018 - Full - true -+ true - - - true -@@ -140,8 +133,8 @@ - Level3 - _OFF_T_DEFINED;WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions);_WIN32_WINNT=0x0501;LUA_ANSI;ENABLE_CJSON_GLOBAL - NotUsing -- MultiThreaded - 4244;4018 -+ true - - - true -diff --git a/src/Win32_Interop/Win32_ANSI.c b/src/Win32_Interop/Win32_ANSI.c -index 404b84f..e7c55d2 ---- a/src/Win32_Interop/Win32_ANSI.c -+++ b/src/Win32_Interop/Win32_ANSI.c -@@ -737,7 +737,7 @@ void ANSI_printf(char *format, ...) { - memset(buffer, 0, cBufLen); - - va_start(args, format); -- retVal = vsprintf_s(buffer, cBufLen, format, args); -+ retVal = vsnprintf(buffer, cBufLen - 1, format, args); - va_end(args); - - if (retVal > 0) { -diff --git a/src/Win32_Interop/Win32_EventLog.cpp b/src/Win32_Interop/Win32_EventLog.cpp -index 1856540..3db4ddd ---- a/src/Win32_Interop/Win32_EventLog.cpp -+++ b/src/Win32_Interop/Win32_EventLog.cpp -@@ -30,7 +30,6 @@ using namespace std; - - #include "Win32_EventLog.h" - #include "Win32_SmartHandle.h" --#include "EventLog.h" - - static bool eventLogEnabled = true; - static string eventLogIdentity = "redis"; -@@ -129,17 +128,17 @@ void RedisEventLog::LogMessage(LPCSTR msg, const WORD type) { - DWORD eventID; - switch (type) { - case EVENTLOG_ERROR_TYPE: -- eventID = MSG_ERROR_1; -+ eventID = 0x2; - break; - case EVENTLOG_WARNING_TYPE: -- eventID = MSG_WARNING_1; -+ eventID = 0x1; - break; - case EVENTLOG_INFORMATION_TYPE: -- eventID = MSG_INFO_1; -+ eventID = 0x0; - break; - default: - std::cerr << "Unrecognized type: " << type << "\n"; -- eventID = MSG_INFO_1; -+ eventID = 0x0; - break; - } - -diff --git a/src/Win32_Interop/Win32_FDAPI.cpp b/src/Win32_Interop/Win32_FDAPI.cpp -index 3df9af1..f60e3d4 ---- a/src/Win32_Interop/Win32_FDAPI.cpp -+++ b/src/Win32_Interop/Win32_FDAPI.cpp -@@ -46,11 +46,13 @@ fdapi_access access = NULL; - fdapi_bind bind = NULL; - fdapi_connect connect = NULL; - fdapi_fcntl fcntl = NULL; -+fdapi_ioctl ioctl = NULL; - fdapi_fstat fdapi_fstat64 = NULL; - fdapi_fsync fsync = NULL; - fdapi_ftruncate ftruncate = NULL; - fdapi_freeaddrinfo freeaddrinfo = NULL; - fdapi_getaddrinfo getaddrinfo = NULL; -+fdapi_gethostbyname gethostbyname = NULL; - fdapi_getpeername getpeername = NULL; - fdapi_getsockname getsockname = NULL; - fdapi_getsockopt getsockopt = NULL; -@@ -67,7 +69,9 @@ fdapi_open open = NULL; - fdapi_pipe pipe = NULL; - fdapi_poll poll = NULL; - fdapi_read read = NULL; -+fdapi_recv recv = NULL; - fdapi_select select = NULL; -+fdapi_send send = NULL; - fdapi_setsockopt setsockopt = NULL; - fdapi_socket socket = NULL; - fdapi_write write = NULL; -@@ -622,6 +626,23 @@ int FDAPI_fcntl(int rfd, int cmd, int flags = 0 ) { - return -1; - } - -+int FDAPI_ioctl(int rfd, int cmd, char *buf) { -+ try { -+ SocketInfo* socket_info = RFDMap::getInstance().lookupSocketInfo(rfd); -+ if (socket_info != NULL && socket_info->socket != INVALID_SOCKET) { -+ if (f_ioctlsocket(socket_info->socket, cmd, (u_long *)buf) != SOCKET_ERROR) { -+ return 0; -+ } else { -+ errno = f_WSAGetLastError(); -+ return -1; -+ } -+ } -+ } CATCH_AND_REPORT(); -+ -+ errno = EBADF; -+ return -1; -+} -+ - int FDAPI_poll(struct pollfd *fds, nfds_t nfds, int timeout) { - try { - struct pollfd* pollCopy = new struct pollfd[nfds]; -@@ -777,6 +798,42 @@ ssize_t FDAPI_read(int rfd, void *buf, size_t count) { - return -1; - } - -+ssize_t FDAPI_recv(int rfd, void *buf, size_t count, int flags) { -+ try { -+ SOCKET socket = RFDMap::getInstance().lookupSocket(rfd); -+ if (socket != INVALID_SOCKET) { -+ int retval = f_recv(socket, (char*) buf, (unsigned int) count, flags); -+ if (retval == -1) { -+ errno = GetLastError(); -+ if (errno == WSAEWOULDBLOCK) { -+ errno = EAGAIN; -+ } -+ } -+ return retval; -+ } -+ } CATCH_AND_REPORT(); -+ errno = EBADF; -+ return -1; -+} -+ -+ssize_t FDAPI_send(int rfd, const void *buf, size_t count, int flags) { -+ try { -+ SOCKET socket = RFDMap::getInstance().lookupSocket(rfd); -+ if (socket != INVALID_SOCKET) { -+ int retval = f_send(socket, (const char*) buf, (unsigned int) count, flags); -+ if (retval == -1) { -+ errno = GetLastError(); -+ if (errno == WSAEWOULDBLOCK) { -+ errno = EAGAIN; -+ } -+ } -+ return retval; -+ } -+ } CATCH_AND_REPORT(); -+ errno = EBADF; -+ return -1; -+} -+ - ssize_t FDAPI_write(int rfd, const void *buf, size_t count) { - try { - SOCKET socket = RFDMap::getInstance().lookupSocket(rfd); -@@ -1195,12 +1252,14 @@ private: - bind = FDAPI_bind; - connect = FDAPI_connect; - fcntl = FDAPI_fcntl; -+ ioctl = FDAPI_ioctl; - fdapi_fstat64 = (fdapi_fstat) FDAPI_fstat64; - freeaddrinfo = FDAPI_freeaddrinfo; - fsync = FDAPI_fsync; - ftruncate = FDAPI_ftruncate; - getaddrinfo = FDAPI_getaddrinfo; - getsockopt = FDAPI_getsockopt; -+ gethostbyname = FDAPI_gethostbyname; - getpeername = FDAPI_getpeername; - getsockname = FDAPI_getsockname; - htonl = FDAPI_htonl; -@@ -1216,9 +1275,11 @@ private: - pipe = FDAPI_pipe; - poll = FDAPI_poll; - read = FDAPI_read; -+ recv = FDAPI_recv; - select = FDAPI_select; - setsockopt = FDAPI_setsockopt; - socket = FDAPI_socket; -+ send = FDAPI_send; - write = FDAPI_write; - } - -diff --git a/src/Win32_Interop/Win32_FDAPI.h b/src/Win32_Interop/Win32_FDAPI.h -index 8fae9c7..6e09596 ---- a/src/Win32_Interop/Win32_FDAPI.h -+++ b/src/Win32_Interop/Win32_FDAPI.h -@@ -116,9 +116,12 @@ typedef int (*fdapi_open)(const char * _Filename, int _OpenFlag, int flags); - typedef int (*fdapi_accept)(int sockfd, struct sockaddr *addr, socklen_t *addrlen); - typedef int (*fdapi_setsockopt)(int sockfd, int level, int optname,const void *optval, socklen_t optlen); - typedef int (*fdapi_fcntl)(int fd, int cmd, int flags); -+typedef int (*fdapi_ioctl)(int fd, int cmd, char *buf); - typedef int (*fdapi_poll)(struct pollfd *fds, nfds_t nfds, int timeout); - typedef int (*fdapi_getsockopt)(int sockfd, int level, int optname, void *optval, socklen_t *optlen); - typedef int (*fdapi_connect)(int sockfd, const struct sockaddr *addr, size_t addrlen); -+typedef ssize_t (*fdapi_recv)(int fd, void *buf, size_t count, int flags); -+typedef ssize_t (*fdapi_send)(int rfd, void const *buf, size_t count, int flags); - typedef ssize_t (*fdapi_read)(int fd, void *buf, size_t count); - typedef ssize_t (*fdapi_write)(int fd, const void *buf, size_t count); - typedef int (*fdapi_fsync)(int fd); -@@ -128,6 +131,7 @@ typedef int (*fdapi_bind)(int sockfd, const struct sockaddr *addr, socklen_t add - typedef u_short (*fdapi_htons)(u_short hostshort); - typedef u_long (*fdapi_htonl)(u_long hostlong); - typedef u_short (*fdapi_ntohs)(u_short netshort); -+typedef struct hostent* (*fdapi_gethostbyname)(const char *name); - typedef int (*fdapi_getpeername)(int sockfd, struct sockaddr *addr, socklen_t * addrlen); - typedef int (*fdapi_getsockname)(int sockfd, struct sockaddr* addrsock, int* addrlen ); - typedef void (*fdapi_freeaddrinfo)(struct addrinfo *ai); -@@ -159,12 +163,14 @@ extern fdapi_access access; - extern fdapi_bind bind; - extern fdapi_connect connect; - extern fdapi_fcntl fcntl; -+extern fdapi_ioctl ioctl; - extern fdapi_fstat fdapi_fstat64; - extern fdapi_freeaddrinfo freeaddrinfo; - extern fdapi_fsync fsync; - extern fdapi_ftruncate ftruncate; - extern fdapi_getaddrinfo getaddrinfo; - extern fdapi_getsockopt getsockopt; -+extern fdapi_gethostbyname gethostbyname; - extern fdapi_getpeername getpeername; - extern fdapi_getsockname getsockname; - extern fdapi_htonl htonl; -@@ -180,7 +186,9 @@ extern fdapi_open open; - extern fdapi_pipe pipe; - extern fdapi_poll poll; - extern fdapi_read read; -+extern fdapi_recv recv; - extern fdapi_select select; -+extern fdapi_send send; - extern fdapi_setsockopt setsockopt; - extern fdapi_socket socket; - extern fdapi_write write; -diff --git a/src/Win32_Interop/Win32_Interop.vcxproj b/src/Win32_Interop/Win32_Interop.vcxproj -index 93fc44b..b75d89b ---- a/src/Win32_Interop/Win32_Interop.vcxproj -+++ b/src/Win32_Interop/Win32_Interop.vcxproj -@@ -74,35 +74,6 @@ - - - -- -- -- Document -- md resources --mc.exe -A -b -c -h . -r resources EventLog.mc --rc.exe -foresources/EventLog.res resources/EventLog.rc --link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll -- -- md resources --mc.exe -A -b -c -h . -r resources EventLog.mc --rc.exe -foresources/EventLog.res resources/EventLog.rc --link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll -- -- EventLog.h -- EventLog.h -- md resources --mc.exe -A -b -c -h . -r resources EventLog.mc --rc.exe -foresources/EventLog.res resources/EventLog.rc --link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll -- -- md resources --mc.exe -A -b -c -h . -r resources EventLog.mc --rc.exe -foresources/EventLog.res resources/EventLog.rc --link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll -- -- EventLog.h -- EventLog.h -- -- - - {8C07F811-C81C-432C-B334-1AE6FAECF951} - Win32Proj -@@ -113,27 +84,25 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - - StaticLibrary - true -- v120 -+ v140_xp - Unicode - - - StaticLibrary - true -- v120 -+ v140_xp - Unicode - - - StaticLibrary - false -- v120 -- true -+ v140_xp - Unicode - - - StaticLibrary - false -- v120 -- true -+ v140_xp - Unicode - - -@@ -152,13 +121,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - - - -- -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -- -- -- $(SolutionDir)$(Platform)\$(Configuration)\ -- $(Platform)\$(Configuration)\ -+ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\ -+ $(SolutionDir)build\$(Platform)\$(Configuration)\$(MSBuildProjectName)\ - - - -@@ -166,9 +131,10 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - - Level3 - Disabled -- USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1 -+ _WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1 - $(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include -- MultiThreadedDebug -+ false -+ true - - - Windows -@@ -186,9 +152,10 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - - Level3 - Disabled -- USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501 -+ _WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501 - $(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include -- MultiThreadedDebug -+ false -+ true - - - Windows -@@ -211,10 +178,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - Full - true - true -- USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1 -+ _WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1 - $(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include -- MultiThreaded -- true -+ true - - - Windows -@@ -235,9 +201,9 @@ link.exe -dll -noentry resources/EventLog.res -out:$(TargetDir)EventLog.dll - MaxSpeed - true - true -- USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501 -+ _WIN32_WINNT=0x0502;USE_STATIC;USE_JEMALLOC;_OFF_T_DEFINED;_NO_CRT_STDIO_INLINE;_CRT_SECURE_NO_DEPRECATE;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions);LACKS_STDLIB_H;_CRT_SECURE_NO_WARNINGS;PSAPI_VERSION=1;_WIN32_WINNT=0x0501 - $(ProjectDir)..\..\deps\lua\src;$(ProjectDir)..\..\deps\jemalloc-win\include -- MultiThreaded -+ true - - - Windows -diff --git a/src/Win32_Interop/Win32_service.cpp b/src/Win32_Interop/Win32_service.cpp -index 488538e..1c33f53 ---- a/src/Win32_Interop/Win32_service.cpp -+++ b/src/Win32_Interop/Win32_service.cpp -@@ -59,7 +59,6 @@ this should preceed the other arguments passed to redis. For instance: - #include - #include - #include --#include - #include - #include "Win32_EventLog.h" - #include -diff --git a/src/ziplist.c b/src/ziplist.c -index 24b0a7c..29d445d ---- a/src/ziplist.c -+++ b/src/ziplist.c -@@ -920,7 +920,7 @@ void ziplistRepr(unsigned char *zl) { - entry = zipEntry(p); - printf( - "{" -- "addr 0x%08lx, " /* TODO" verify 0x%08lx */ -+ "addr %p, " - "index %2d, " - "offset %5ld, " - "rl: %5u, " -@@ -929,9 +929,9 @@ void ziplistRepr(unsigned char *zl) { - "pls: %2u, " - "payload %5u" - "} ", -- (PORT_ULONG)p, -+ (void *)p, - index, -- (PORT_ULONG)(p-zl), -+ (long)(p-zl), - entry.headersize+entry.len, - entry.headersize, - entry.prevrawlen, diff --git a/src/global_scheduler/CMakeLists.txt b/src/global_scheduler/CMakeLists.txt deleted file mode 100644 index fec7ec281..000000000 --- a/src/global_scheduler/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -cmake_minimum_required(VERSION 3.4) - -project(global_scheduler) - -include_directories(${CMAKE_CURRENT_LIST_DIR}) - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") - -add_executable(global_scheduler global_scheduler.cc global_scheduler_algorithm.cc) - -# Make sure ${HIREDIS_LIB} is ready before linking. -add_dependencies(global_scheduler hiredis common) - -target_link_libraries(global_scheduler common ${HIREDIS_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} pthread) diff --git a/src/global_scheduler/global_scheduler.cc b/src/global_scheduler/global_scheduler.cc deleted file mode 100644 index d964401ae..000000000 --- a/src/global_scheduler/global_scheduler.cc +++ /dev/null @@ -1,492 +0,0 @@ -#include -#include -#include - -#include "common.h" -#include "event_loop.h" -#include "global_scheduler.h" -#include "global_scheduler_algorithm.h" -#include "net.h" -#include "ray/util/util.h" -#include "state/db_client_table.h" -#include "state/local_scheduler_table.h" -#include "state/object_table.h" -#include "state/table.h" -#include "state/task_table.h" - -/** - * Retry the task assignment. If the local scheduler that the task is assigned - * to is no longer active, do not retry the assignment. - * TODO(rkn): We currently only retry the method if the global scheduler - * publishes a task to a local scheduler before the local scheduler has - * subscribed to the channel. If we enforce that ordering, we can remove this - * retry method. - * - * @param id The task ID. - * @param user_context The global scheduler state. - * @param user_data The Task that failed to be assigned. - * @return Void. - */ -void assign_task_to_local_scheduler_retry(UniqueID id, - void *user_context, - void *user_data) { - GlobalSchedulerState *state = (GlobalSchedulerState *) user_context; - Task *task = (Task *) user_data; - RAY_CHECK(Task_state(task) == TaskStatus::SCHEDULED); - - // If the local scheduler has died since we requested the task assignment, do - // not retry again. - DBClientID local_scheduler_id = Task_local_scheduler(task); - auto it = state->local_schedulers.find(local_scheduler_id); - if (it == state->local_schedulers.end()) { - return; - } - - // The local scheduler is still alive. The failure is most likely due to the - // task assignment getting published before the local scheduler subscribed to - // the channel. Retry the assignment. - auto retryInfo = RetryInfo{ - .num_retries = 0, // This value is unused. - .timeout = 0, // This value is unused. - .fail_callback = assign_task_to_local_scheduler_retry, - }; - task_table_update(state->db, Task_copy(task), &retryInfo, NULL, user_context); -} - -/** - * Assign the given task to the local scheduler, update Redis and scheduler data - * structures. - * - * @param state Global scheduler state. - * @param task Task to be assigned to the local scheduler. - * @param local_scheduler_id DB client ID for the local scheduler. - * @return Void. - */ -void assign_task_to_local_scheduler(GlobalSchedulerState *state, - Task *task, - DBClientID local_scheduler_id) { - TaskSpec *spec = Task_task_execution_spec(task)->Spec(); - RAY_LOG(DEBUG) << "assigning task to local_scheduler_id = " - << local_scheduler_id; - Task_set_state(task, TaskStatus::SCHEDULED); - Task_set_local_scheduler(task, local_scheduler_id); - RAY_LOG(DEBUG) << "Issuing a task table update for task = " - << Task_task_id(task); - - auto retryInfo = RetryInfo{ - .num_retries = 0, // This value is unused. - .timeout = 0, // This value is unused. - .fail_callback = assign_task_to_local_scheduler_retry, - }; - task_table_update(state->db, Task_copy(task), &retryInfo, NULL, state); - - /* Update the object table info to reflect the fact that the results of this - * task will be created on the machine that the task was assigned to. This can - * be used to improve locality-aware scheduling. */ - for (int64_t i = 0; i < TaskSpec_num_returns(spec); ++i) { - ObjectID return_id = TaskSpec_return(spec, i); - if (state->scheduler_object_info_table.find(return_id) == - state->scheduler_object_info_table.end()) { - SchedulerObjectInfo &obj_info_entry = - state->scheduler_object_info_table[return_id]; - /* The value -1 indicates that the size of the object is not known yet. */ - obj_info_entry.data_size = -1; - } - RAY_CHECK(state->local_scheduler_plasma_map.count(local_scheduler_id) == 1); - state->scheduler_object_info_table[return_id].object_locations.push_back( - state->local_scheduler_plasma_map[local_scheduler_id]); - } - - /* TODO(rkn): We should probably pass around local_scheduler struct pointers - * instead of db_client_id objects. */ - /* Update the local scheduler info. */ - auto it = state->local_schedulers.find(local_scheduler_id); - RAY_CHECK(it != state->local_schedulers.end()); - - LocalScheduler &local_scheduler = it->second; - local_scheduler.num_tasks_sent += 1; - local_scheduler.num_recent_tasks_sent += 1; - // Resource accounting update for this local scheduler. - for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) { - std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - // The local scheduler must have this resource because otherwise we wouldn't - // be assigning the task to this local scheduler. - RAY_CHECK(local_scheduler.info.dynamic_resources.count(resource_name) == - 1 || - resource_quantity == 0); - // Subtract task's resource from the cached dynamic resource capacity for - // this local scheduler. This will be overwritten on the next heartbeat. - local_scheduler.info.dynamic_resources[resource_name] = - MAX(0, local_scheduler.info.dynamic_resources[resource_name] - - resource_quantity); - } -} - -GlobalSchedulerState *GlobalSchedulerState_init(event_loop *loop, - const char *node_ip_address, - const char *redis_primary_addr, - int redis_primary_port) { - GlobalSchedulerState *state = new GlobalSchedulerState(); - state->loop = loop; - state->db = db_connect(std::string(redis_primary_addr), redis_primary_port, - "global_scheduler", node_ip_address, - std::vector()); - db_attach(state->db, loop, false); - state->policy_state = GlobalSchedulerPolicyState_init(); - return state; -} - -void GlobalSchedulerState_free(GlobalSchedulerState *state) { - db_disconnect(state->db); - state->local_schedulers.clear(); - GlobalSchedulerPolicyState_free(state->policy_state); - /* Delete the plasma to local scheduler association map. */ - state->plasma_local_scheduler_map.clear(); - - /* Delete the local scheduler to plasma association map. */ - state->local_scheduler_plasma_map.clear(); - - /* Free the scheduler object info table. */ - state->scheduler_object_info_table.clear(); - /* Free the array of unschedulable tasks. */ - int64_t num_pending_tasks = state->pending_tasks.size(); - if (num_pending_tasks > 0) { - RAY_LOG(WARNING) << "There are " << num_pending_tasks - << " remaining tasks in the pending tasks array."; - } - for (int i = 0; i < num_pending_tasks; ++i) { - Task *pending_task = state->pending_tasks[i]; - Task_free(pending_task); - } - state->pending_tasks.clear(); - - /* Destroy the event loop. */ - destroy_outstanding_callbacks(state->loop); - event_loop_destroy(state->loop); - state->loop = NULL; - - /* Free the global scheduler state. */ - delete state; -} - -/* We need this code so we can clean up when we get a SIGTERM signal. */ - -GlobalSchedulerState *g_state; - -void signal_handler(int signal) { - if (signal == SIGTERM) { - GlobalSchedulerState_free(g_state); - exit(0); - } -} - -/* End of the cleanup code. */ - -void process_task_waiting(Task *waiting_task, void *user_context) { - GlobalSchedulerState *state = (GlobalSchedulerState *) user_context; - RAY_LOG(DEBUG) << "Task waiting callback is called."; - bool successfully_assigned = - handle_task_waiting(state, state->policy_state, waiting_task); - /* If the task was not successfully submitted to a local scheduler, add the - * task to the array of pending tasks. The global scheduler will periodically - * resubmit the tasks in this array. */ - if (!successfully_assigned) { - Task *task_copy = Task_copy(waiting_task); - state->pending_tasks.push_back(task_copy); - } -} - -void add_local_scheduler(GlobalSchedulerState *state, - DBClientID db_client_id, - const char *manager_address) { - /* Add plasma_manager ip:port -> local_scheduler_db_client_id association to - * state. */ - state->plasma_local_scheduler_map[std::string(manager_address)] = - db_client_id; - - /* Add local_scheduler_db_client_id -> plasma_manager ip:port association to - * state. */ - state->local_scheduler_plasma_map[db_client_id] = - std::string(manager_address); - - /* Add new local scheduler to the state. */ - LocalScheduler &local_scheduler = state->local_schedulers[db_client_id]; - local_scheduler.id = db_client_id; - local_scheduler.num_heartbeats_missed = 0; - local_scheduler.num_tasks_sent = 0; - local_scheduler.num_recent_tasks_sent = 0; - local_scheduler.info.task_queue_length = 0; - local_scheduler.info.available_workers = 0; - - /* Allow the scheduling algorithm to process this event. */ - handle_new_local_scheduler(state, state->policy_state, db_client_id); -} - -std::unordered_map::iterator remove_local_scheduler( - GlobalSchedulerState *state, - std::unordered_map::iterator it) { - RAY_CHECK(it != state->local_schedulers.end()); - DBClientID local_scheduler_id = it->first; - it = state->local_schedulers.erase(it); - - /* Remove the local scheduler from the mappings. This code only makes sense if - * there is a one-to-one mapping between local schedulers and plasma managers. - */ - std::string manager_address = - state->local_scheduler_plasma_map[local_scheduler_id]; - state->local_scheduler_plasma_map.erase(local_scheduler_id); - state->plasma_local_scheduler_map.erase(manager_address); - - handle_local_scheduler_removed(state, state->policy_state, - local_scheduler_id); - return it; -} - -/** - * Process a notification about a new DB client connecting to Redis. - * - * @param manager_address An ip:port pair for the plasma manager associated with - * this db client. - * @return Void. - */ -void process_new_db_client(DBClient *db_client, void *user_context) { - GlobalSchedulerState *state = (GlobalSchedulerState *) user_context; - RAY_LOG(DEBUG) << "db client table callback for db client = " - << db_client->id; - if (strncmp(db_client->client_type.c_str(), "local_scheduler", - strlen("local_scheduler")) == 0) { - bool local_scheduler_present = - (state->local_schedulers.find(db_client->id) != - state->local_schedulers.end()); - if (db_client->is_alive) { - /* This is a notification for an insert. We may receive duplicate - * notifications since we read the entire table before processing - * notifications. Filter out local schedulers that we already added. */ - if (!local_scheduler_present) { - add_local_scheduler(state, db_client->id, - db_client->manager_address.c_str()); - } - } else { - if (local_scheduler_present) { - remove_local_scheduler(state, - state->local_schedulers.find(db_client->id)); - } - } - } -} - -/** - * Process notification about the new object information. - * - * @param object_id ID of the object that the notification is about. - * @param data_size The object size. - * @param manager_count The number of locations for this object. - * @param manager_ids The vector of Plasma Manager client IDs. - * @param user_context The user context. - * @return Void. - */ -void object_table_subscribe_callback(ObjectID object_id, - int64_t data_size, - const std::vector &manager_ids, - void *user_context) { - /* Extract global scheduler state from the callback context. */ - GlobalSchedulerState *state = (GlobalSchedulerState *) user_context; - RAY_LOG(DEBUG) << "object table subscribe callback for OBJECT = " - << object_id; - - const std::vector managers = - db_client_table_get_ip_addresses(state->db, manager_ids); - RAY_LOG(DEBUG) << "\tManagers<" << managers.size() << ">:"; - for (size_t i = 0; i < managers.size(); i++) { - RAY_LOG(DEBUG) << "\t\t" << managers[i]; - } - - if (state->scheduler_object_info_table.find(object_id) == - state->scheduler_object_info_table.end()) { - /* Construct a new object info hash table entry. */ - SchedulerObjectInfo &obj_info_entry = - state->scheduler_object_info_table[object_id]; - obj_info_entry.data_size = data_size; - - RAY_LOG(DEBUG) << "New object added to object_info_table with id = " - << object_id; - RAY_LOG(DEBUG) << "\tmanager locations:"; - for (size_t i = 0; i < managers.size(); i++) { - RAY_LOG(DEBUG) << "\t\t" << managers[i]; - } - } - - SchedulerObjectInfo &obj_info_entry = - state->scheduler_object_info_table[object_id]; - - /* In all cases, replace the object location vector on each callback. */ - obj_info_entry.object_locations.clear(); - for (size_t i = 0; i < managers.size(); i++) { - obj_info_entry.object_locations.push_back(managers[i]); - } -} - -void local_scheduler_table_handler(DBClientID client_id, - LocalSchedulerInfo info, - void *user_context) { - /* Extract global scheduler state from the callback context. */ - GlobalSchedulerState *state = (GlobalSchedulerState *) user_context; - ARROW_UNUSED(state); - RAY_LOG(DEBUG) << "Local scheduler heartbeat from db_client_id " << client_id; - RAY_LOG(DEBUG) << "total workers = " << info.total_num_workers - << ", task queue length = " << info.task_queue_length - << ", available workers = " << info.available_workers; - - /* Update the local scheduler info struct. */ - auto it = state->local_schedulers.find(client_id); - if (it != state->local_schedulers.end()) { - if (info.is_dead) { - /* The local scheduler is exiting. Increase the number of heartbeats - * missed to the timeout threshold. This will trigger removal of the - * local scheduler the next time the timeout handler fires. */ - it->second.num_heartbeats_missed = - RayConfig::instance().num_heartbeats_timeout(); - } else { - /* Reset the number of tasks sent since the last heartbeat. */ - LocalScheduler &local_scheduler = it->second; - local_scheduler.num_heartbeats_missed = 0; - local_scheduler.num_recent_tasks_sent = 0; - local_scheduler.info = info; - } - } else { - RAY_LOG(WARNING) << "client_id didn't match any cached local scheduler " - << "entries"; - } -} - -int task_cleanup_handler(event_loop *loop, timer_id id, void *context) { - GlobalSchedulerState *state = (GlobalSchedulerState *) context; - /* Loop over the pending tasks in reverse order and resubmit them. */ - auto it = state->pending_tasks.end(); - while (it != state->pending_tasks.begin()) { - it--; - Task *pending_task = *it; - /* Pretend that the task has been resubmitted. */ - bool successfully_assigned = - handle_task_waiting(state, state->policy_state, pending_task); - if (successfully_assigned) { - /* The task was successfully assigned, so remove it from this list and - * free it. This uses the fact that pending_tasks is a vector and so erase - * returns an iterator to the next element in the vector. */ - it = state->pending_tasks.erase(it); - Task_free(pending_task); - } - } - - return GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS; -} - -int heartbeat_timeout_handler(event_loop *loop, timer_id id, void *context) { - GlobalSchedulerState *state = (GlobalSchedulerState *) context; - /* Check for local schedulers that have missed a number of heartbeats. If any - * local schedulers have died, notify others so that the state can be cleaned - * up. */ - /* TODO(swang): If the local scheduler hasn't actually died, then it should - * clean up its state and exit upon receiving this notification. */ - auto it = state->local_schedulers.begin(); - while (it != state->local_schedulers.end()) { - if (it->second.num_heartbeats_missed >= - RayConfig::instance().num_heartbeats_timeout()) { - RAY_LOG(WARNING) << "Missed too many heartbeats from local scheduler, " - << "marking as dead."; - /* Notify others by updating the global state. */ - db_client_table_remove(state->db, it->second.id, NULL, NULL, NULL); - /* Remove the scheduler from the local state. The call to - * remove_local_scheduler modifies the container in place and returns the - * next iterator. */ - it = remove_local_scheduler(state, it); - } else { - it->second.num_heartbeats_missed += 1; - it++; - } - } - - /* Reset the timer. */ - return RayConfig::instance().heartbeat_timeout_milliseconds(); -} - -void start_server(const char *node_ip_address, - const char *redis_primary_addr, - int redis_primary_port) { - event_loop *loop = event_loop_create(); - g_state = GlobalSchedulerState_init(loop, node_ip_address, redis_primary_addr, - redis_primary_port); - /* TODO(rkn): subscribe to notifications from the object table. */ - /* Subscribe to notifications about new local schedulers. TODO(rkn): this - * needs to also get all of the clients that registered with the database - * before this call to subscribe. */ - db_client_table_subscribe(g_state->db, process_new_db_client, - (void *) g_state, NULL, NULL, NULL); - /* Subscribe to notifications about waiting tasks. If a local scheduler - * submits tasks to the global scheduler before the global scheduler - * successfully subscribes, then the local scheduler that submitted the tasks - * will retry. */ - task_table_subscribe(g_state->db, UniqueID::nil(), TaskStatus::WAITING, - process_task_waiting, (void *) g_state, NULL, NULL, - NULL); - - object_table_subscribe_to_notifications(g_state->db, true, - object_table_subscribe_callback, - g_state, NULL, NULL, NULL); - /* Subscribe to notifications from local schedulers. These notifications serve - * as heartbeats and contain informaion about the load on the local - * schedulers. */ - local_scheduler_table_subscribe(g_state->db, local_scheduler_table_handler, - g_state, NULL); - /* Start a timer that periodically checks if there are queued tasks that can - * be scheduled. Currently this is only used to handle the special case in - * which a task is waiting and no node meets its static resource requirements. - * If a new node joins the cluster that does have enough resources, then this - * timer should notice and schedule the task. */ - event_loop_add_timer(loop, GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS, - task_cleanup_handler, g_state); - event_loop_add_timer(loop, - RayConfig::instance().heartbeat_timeout_milliseconds(), - heartbeat_timeout_handler, g_state); - /* Start the event loop. */ - event_loop_run(loop); -} - -int main(int argc, char *argv[]) { - InitShutdownRAII ray_log_shutdown_raii( - ray::RayLog::StartRayLog, ray::RayLog::ShutDownRayLog, argv[0], - ray::RayLogLevel::INFO, /*log_dir=*/""); - ray::RayLog::InstallFailureSignalHandler(); - signal(SIGTERM, signal_handler); - /* IP address and port of the primary redis instance. */ - char *redis_primary_addr_port = NULL; - /* The IP address of the node that this global scheduler is running on. */ - char *node_ip_address = NULL; - int c; - while ((c = getopt(argc, argv, "h:r:")) != -1) { - switch (c) { - case 'r': - redis_primary_addr_port = optarg; - break; - case 'h': - node_ip_address = optarg; - break; - default: - RAY_LOG(FATAL) << "unknown option " << c; - } - } - - char redis_primary_addr[16]; - int redis_primary_port = -1; - if (!redis_primary_addr_port || - parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr, - &redis_primary_port) == -1) { - RAY_LOG(FATAL) << "specify the primary redis address like 127.0.0.1:6379 " - << "with the -r switch"; - } - if (!node_ip_address) { - RAY_LOG(FATAL) << "specify the node IP address with the -h switch"; - } - start_server(node_ip_address, redis_primary_addr, redis_primary_port); -} diff --git a/src/global_scheduler/global_scheduler.h b/src/global_scheduler/global_scheduler.h deleted file mode 100644 index e1610c555..000000000 --- a/src/global_scheduler/global_scheduler.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef GLOBAL_SCHEDULER_H -#define GLOBAL_SCHEDULER_H - -#include "task.h" - -#include - -#include "ray/gcs/client.h" -#include "state/db.h" -#include "state/local_scheduler_table.h" - -/* The frequency with which the global scheduler checks if there are any tasks - * that haven't been scheduled yet. */ -#define GLOBAL_SCHEDULER_TASK_CLEANUP_MILLISECONDS 100 - -/** Contains all information that is associated with a local scheduler. */ -typedef struct { - /** The ID of the local scheduler in Redis. */ - DBClientID id; - /** The number of heartbeat intervals that have passed since we last heard - * from this local scheduler. */ - int64_t num_heartbeats_missed; - /** The number of tasks sent from the global scheduler to this local - * scheduler. */ - int64_t num_tasks_sent; - /** The number of tasks sent from the global scheduler to this local scheduler - * since the last heartbeat arrived. */ - int64_t num_recent_tasks_sent; - /** The latest information about the local scheduler capacity. This is updated - * every time a new local scheduler heartbeat arrives. */ - LocalSchedulerInfo info; -} LocalScheduler; - -typedef class GlobalSchedulerPolicyState GlobalSchedulerPolicyState; - -/** - * This defines a hash table used to cache information about different objects. - */ -typedef struct { - /** The size in bytes of the object. */ - int64_t data_size; - /** A vector of object locations for this object. */ - std::vector object_locations; -} SchedulerObjectInfo; - -/** - * Global scheduler state structure. - */ -typedef struct { - /** The global scheduler event loop. */ - event_loop *loop; - /** The global state store database. */ - DBHandle *db; - /** A hash table mapping local scheduler ID to the local schedulers that are - * connected to Redis. */ - std::unordered_map local_schedulers; - /** The state managed by the scheduling policy. */ - GlobalSchedulerPolicyState *policy_state; - /** The plasma_manager ip:port -> local_scheduler_db_client_id association. */ - std::unordered_map plasma_local_scheduler_map; - /** The local_scheduler_db_client_id -> plasma_manager ip:port association. */ - std::unordered_map local_scheduler_plasma_map; - /** Objects cached by this global scheduler instance. */ - std::unordered_map scheduler_object_info_table; - /** An array of tasks that haven't been scheduled yet. */ - std::vector pending_tasks; -} GlobalSchedulerState; - -/** - * This is a helper method to look up the local scheduler struct that - * corresponds to a particular local_scheduler_id. - * - * @param state The state of the global scheduler. - * @param The local_scheduler_id of the local scheduler. - * @return The corresponding local scheduler struct. If the global scheduler is - * not aware of the local scheduler, then this will be NULL. - */ -LocalScheduler *get_local_scheduler(GlobalSchedulerState *state, - DBClientID local_scheduler_id); - -/** - * Assign the given task to the local scheduler, update Redis and scheduler data - * structures. - * - * @param state Global scheduler state. - * @param task Task to be assigned to the local scheduler. - * @param local_scheduler_id DB client ID for the local scheduler. - * @return Void. - */ -void assign_task_to_local_scheduler(GlobalSchedulerState *state, - Task *task, - DBClientID local_scheduler_id); - -#endif /* GLOBAL_SCHEDULER_H */ diff --git a/src/global_scheduler/global_scheduler_algorithm.cc b/src/global_scheduler/global_scheduler_algorithm.cc deleted file mode 100644 index 7ca1b86be..000000000 --- a/src/global_scheduler/global_scheduler_algorithm.cc +++ /dev/null @@ -1,257 +0,0 @@ -#include - -#include "task.h" -#include "state/task_table.h" - -#include "global_scheduler_algorithm.h" - -GlobalSchedulerPolicyState *GlobalSchedulerPolicyState_init(void) { - GlobalSchedulerPolicyState *policy_state = new GlobalSchedulerPolicyState(); - return policy_state; -} - -void GlobalSchedulerPolicyState_free(GlobalSchedulerPolicyState *policy_state) { - delete policy_state; -} - -/** - * Checks if the given local scheduler satisfies the task's hard constraints. - * - * @param scheduler Local scheduler. - * @param spec Task specification. - * @return True if all tasks's resource constraints are satisfied. False - * otherwise. - */ -bool constraints_satisfied_hard(const LocalScheduler *scheduler, - const TaskSpec *spec) { - if (scheduler->info.static_resources.count("CPU") == 1 && - scheduler->info.static_resources.at("CPU") == 0) { - // Don't give tasks to local schedulers that have 0 CPUs. This can be an - // issue for actor creation tasks that require 0 CPUs (but the subsequent - // actor methods require some CPUs). - return false; - } - - for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) { - std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - - // Continue on if the task doesn't actually require this resource. - if (resource_quantity == 0) { - continue; - } - - // Check if the local scheduler has this resource. - if (scheduler->info.static_resources.count(resource_name) == 0) { - return false; - } - - // Check if the local scheduler has enough of the resource. - if (scheduler->info.static_resources.at(resource_name) < - resource_quantity) { - return false; - } - } - return true; -} - -int64_t locally_available_data_size(const GlobalSchedulerState *state, - DBClientID local_scheduler_id, - TaskSpec *task_spec) { - /* This function will compute the total size of all the object dependencies - * for the given task that are already locally available to the specified - * local scheduler. */ - int64_t task_data_size = 0; - - RAY_CHECK(state->local_scheduler_plasma_map.count(local_scheduler_id) == 1); - - const std::string &plasma_manager = - state->local_scheduler_plasma_map.at(local_scheduler_id); - - /* TODO(rkn): Note that if the same object ID appears as multiple arguments, - * then it will be overcounted. */ - for (int64_t i = 0; i < TaskSpec_num_args(task_spec); ++i) { - int count = TaskSpec_arg_id_count(task_spec, i); - for (int j = 0; j < count; ++j) { - ObjectID object_id = TaskSpec_arg_id(task_spec, i, j); - - if (state->scheduler_object_info_table.count(object_id) == 0) { - /* If this global scheduler is not aware of this object ID, then ignore - * it. */ - continue; - } - - const SchedulerObjectInfo &object_size_info = - state->scheduler_object_info_table.at(object_id); - - if (std::find(object_size_info.object_locations.begin(), - object_size_info.object_locations.end(), plasma_manager) == - object_size_info.object_locations.end()) { - /* This local scheduler does not have access to this object, so don't - * count this object. */ - continue; - } - - /* Look at the size of the object. */ - int64_t object_size = object_size_info.data_size; - if (object_size == -1) { - /* This means that this global scheduler does not know the object size - * yet, so assume that the object is one megabyte. TODO(rkn): Maybe we - * should instead use the average object size. */ - object_size = 1000000; - } - - /* If we get here, then this local scheduler has access to this object, so - * count the contribution of this object. */ - task_data_size += object_size; - } - } - - return task_data_size; -} - -double calculate_cost_pending(const GlobalSchedulerState *state, - const LocalScheduler *scheduler, - TaskSpec *task_spec) { - /* Calculate how much data is already present on this machine. TODO(rkn): Note - * that this information is not being used yet. Fix this. */ - locally_available_data_size(state, scheduler->id, task_spec); - /* TODO(rkn): This logic does not load balance properly when the different - * machines have different sizes. Fix this. */ - double cost_pending = scheduler->num_recent_tasks_sent + - scheduler->info.task_queue_length - - scheduler->info.available_workers; - return cost_pending; -} - -bool handle_task_waiting_random(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - Task *task) { - TaskSpec *task_spec = Task_task_execution_spec(task)->Spec(); - RAY_CHECK(task_spec != NULL) - << "task wait handler encounted a task with NULL spec"; - - std::vector feasible_nodes; - - for (const auto &it : state->local_schedulers) { - // Local scheduler map iterator yields pairs. - const LocalScheduler &local_scheduler = it.second; - if (!constraints_satisfied_hard(&local_scheduler, task_spec)) { - continue; - } - // Add this local scheduler as a candidate for random selection. - feasible_nodes.push_back(it.first); - } - - if (feasible_nodes.size() == 0) { - RAY_LOG(ERROR) << "Infeasible task. No nodes satisfy hard constraints for " - << "task = " << Task_task_id(task); - return false; - } - - // Randomly select the local scheduler. TODO(atumanov): replace with - // std::discrete_distribution. - std::uniform_int_distribution<> dis(0, feasible_nodes.size() - 1); - DBClientID local_scheduler_id = - feasible_nodes[dis(policy_state->getRandomGenerator())]; - RAY_CHECK(!local_scheduler_id.is_nil()) - << "Task is feasible, but doesn't have a local scheduler assigned."; - // A local scheduler ID was found, so assign the task. - assign_task_to_local_scheduler(state, task, local_scheduler_id); - return true; -} - -bool handle_task_waiting_cost(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - Task *task) { - TaskSpec *task_spec = Task_task_execution_spec(task)->Spec(); - int64_t curtime = current_time_ms(); - - RAY_CHECK(task_spec != NULL) - << "task wait handler encounted a task with NULL spec"; - - // For tasks already seen by the global scheduler (spillback > 1), - // adjust scheduled task counts for the source local scheduler. - if (task->execution_spec->SpillbackCount() > 1) { - auto it = state->local_schedulers.find(task->local_scheduler_id); - // Task's previous local scheduler must be present and known. - RAY_CHECK(it != state->local_schedulers.end()); - LocalScheduler &src_local_scheduler = it->second; - src_local_scheduler.num_recent_tasks_sent -= 1; - } - - bool task_feasible = false; - - // Go through all the nodes, calculate the score for each, pick max score. - double best_local_scheduler_score = INT32_MIN; - RAY_CHECK(best_local_scheduler_score < 0) - << "We might have a floating point underflow"; - RAY_LOG(INFO) << "ct[" << curtime << "] task from " - << task->local_scheduler_id << " spillback " - << task->execution_spec->SpillbackCount(); - - // The best node to send this task. - DBClientID best_local_scheduler_id = DBClientID::nil(); - - for (auto it = state->local_schedulers.begin(); - it != state->local_schedulers.end(); it++) { - // For each local scheduler, calculate its score. Check hard constraints - // first. - LocalScheduler *scheduler = &(it->second); - if (!constraints_satisfied_hard(scheduler, task_spec)) { - continue; - } - // Skip the local scheduler the task came from. - if (task->local_scheduler_id == scheduler->id) { - continue; - } - task_feasible = true; - // This node satisfies the hard capacity constraint. Calculate its score. - double score = -1 * calculate_cost_pending(state, scheduler, task_spec); - RAY_LOG(INFO) << "ct[" << curtime << "][" << scheduler->id << "][q" - << scheduler->info.task_queue_length << "][w" - << scheduler->info.available_workers << "]: score " << score - << " bestscore " << best_local_scheduler_score; - if (score >= best_local_scheduler_score) { - best_local_scheduler_score = score; - best_local_scheduler_id = scheduler->id; - } - } - - if (!task_feasible) { - RAY_LOG(ERROR) << "Infeasible task. No nodes satisfy hard constraints for " - << "task = " << Task_task_id(task); - // TODO(atumanov): propagate this error to the task's driver and/or - // cache the task in case new local schedulers satisfy it in the future. - return false; - } - RAY_CHECK(!best_local_scheduler_id.is_nil()) - << "Task is feasible, but doesn't have a local scheduler assigned."; - // A local scheduler ID was found, so assign the task. - assign_task_to_local_scheduler(state, task, best_local_scheduler_id); - return true; -} - -bool handle_task_waiting(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - Task *task) { - return handle_task_waiting_random(state, policy_state, task); -} - -void handle_object_available(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - ObjectID object_id) { - /* Do nothing for now. */ -} - -void handle_new_local_scheduler(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - DBClientID db_client_id) { - /* Do nothing for now. */ -} - -void handle_local_scheduler_removed(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - DBClientID db_client_id) { - /* Do nothing for now. */ -} diff --git a/src/global_scheduler/global_scheduler_algorithm.h b/src/global_scheduler/global_scheduler_algorithm.h deleted file mode 100644 index 69be67d97..000000000 --- a/src/global_scheduler/global_scheduler_algorithm.h +++ /dev/null @@ -1,126 +0,0 @@ -#ifndef GLOBAL_SCHEDULER_ALGORITHM_H -#define GLOBAL_SCHEDULER_ALGORITHM_H - -#include -#include - -#include "common.h" -#include "global_scheduler.h" -#include "task.h" - -/* ==== The scheduling algorithm ==== - * - * This file contains declaration for all functions and data structures that - * need to be provided if you want to implement a new algorithm for the global - * scheduler. - * - */ - -enum class GlobalSchedulerAlgorithm { - SCHED_ALGORITHM_ROUND_ROBIN = 1, - SCHED_ALGORITHM_TRANSFER_AWARE = 2, - SCHED_ALGORITHM_MAX -}; - -/// The class encapsulating state managed by the global scheduling policy. -class GlobalSchedulerPolicyState { - public: - GlobalSchedulerPolicyState(int64_t round_robin_index) - : round_robin_index_(round_robin_index), - gen_(std::chrono::high_resolution_clock::now() - .time_since_epoch() - .count()) {} - - GlobalSchedulerPolicyState() - : round_robin_index_(0), - gen_(std::chrono::high_resolution_clock::now() - .time_since_epoch() - .count()) {} - - /// Return the policy's random number generator. - /// - /// @return The policy's random number generator. - std::mt19937_64 &getRandomGenerator() { return gen_; } - - /// Return the round robin index maintained by policy state. - /// - /// @return The round robin index. - int64_t getRoundRobinIndex() const { return round_robin_index_; } - - private: - /// The index of the next local scheduler to assign a task to. - int64_t round_robin_index_; - /// Internally maintained random number generator. - std::mt19937_64 gen_; -}; - -/** - * Create the state of the global scheduler policy. This state must be freed by - * the caller. - * - * @return The state of the scheduling policy. - */ -GlobalSchedulerPolicyState *GlobalSchedulerPolicyState_init(void); - -/** - * Free the global scheduler policy state. - * - * @param policy_state The policy state to free. - * @return Void. - */ -void GlobalSchedulerPolicyState_free(GlobalSchedulerPolicyState *policy_state); - -/** - * Main new task handling function in the global scheduler. - * - * @param state Global scheduler state. - * @param policy_state State specific to the scheduling policy. - * @param task New task to be scheduled. - * @return True if the task was assigned to a local scheduler and false - * otherwise. - */ -bool handle_task_waiting(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - Task *task); - -/** - * Handle the fact that a new object is available. - * - * @param state The global scheduler state. - * @param policy_state The state managed by the scheduling policy. - * @param object_id The ID of the object that is now available. - * @return Void. - */ -void handle_object_available(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - ObjectID object_id); - -/** - * Handle a heartbeat message from a local scheduler. TODO(rkn): this is a - * placeholder for now. - * - * @param state The global scheduler state. - * @param policy_state The state managed by the scheduling policy. - * @return Void. - */ -void handle_local_scheduler_heartbeat(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state); - -/** - * Handle the presence of a new local scheduler. Currently, this just adds the - * local scheduler to a queue of local schedulers. - * - * @param state The global scheduler state. - * @param policy_state The state managed by the scheduling policy. - * @param The db client ID of the new local scheduler. - * @return Void. - */ -void handle_new_local_scheduler(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - DBClientID db_client_id); - -void handle_local_scheduler_removed(GlobalSchedulerState *state, - GlobalSchedulerPolicyState *policy_state, - DBClientID db_client_id); - -#endif /* GLOBAL_SCHEDULER_ALGORITHM_H */ diff --git a/src/local_scheduler/CMakeLists.txt b/src/local_scheduler/CMakeLists.txt deleted file mode 100644 index 7033c4f23..000000000 --- a/src/local_scheduler/CMakeLists.txt +++ /dev/null @@ -1,104 +0,0 @@ -cmake_minimum_required(VERSION 3.4) - -project(local_scheduler) - -add_definitions(-fPIC) - -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - include_directories("${PYTHON_INCLUDE_DIRS}") - include_directories("${NUMPY_INCLUDE_DIR}") -endif() - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall") - -if(UNIX AND NOT APPLE) - link_libraries(rt) -endif() - -include_directories("${CMAKE_CURRENT_LIST_DIR}/") -include_directories("${CMAKE_CURRENT_LIST_DIR}/../") -# TODO(pcm): get rid of this: -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/") -endif() - -include_directories("${ARROW_INCLUDE_DIR}") -include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/") - -# Compile flatbuffers - -set(LOCAL_SCHEDULER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/local_scheduler.fbs") -set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/) - -set(LOCAL_SCHEDULER_FBS_OUTPUT_FILES - "${OUTPUT_DIR}/local_scheduler_generated.h") - -add_custom_command( - OUTPUT ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC} --gen-object-api --scoped-enums - DEPENDS ${FBS_DEPENDS} - COMMENT "Running flatc compiler on ${LOCAL_SCHEDULER_FBS_SRC}" - VERBATIM) - -add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES}) - -add_dependencies(gen_local_scheduler_fbs arrow) - -add_library(local_scheduler_client STATIC local_scheduler_client.cc) - -# local_scheduler_shared.h includes ray/gcs/client.h which requires gen_gcs_fbs & gen_node_manager_fbs. -add_dependencies(local_scheduler_client common hiredis gen_local_scheduler_fbs ${COMMON_FBS_OUTPUT_FILES} gen_gcs_fbs gen_node_manager_fbs) - -add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc) -add_dependencies(local_scheduler hiredis) -target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY}) - -add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_scheduler.cc local_scheduler_algorithm.cc) -add_dependencies(local_scheduler_tests hiredis) -target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY}) -target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST") - -macro(get_local_scheduler_library LANG VAR) - set(${VAR} "local_scheduler_library_${LANG}") -endmacro() - -macro(set_local_scheduler_library LANG) - get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_${LANG}) - set(LOCAL_SCHEDULER_LIBRARY_LANG ${LOCAL_SCHEDULER_LIBRARY_${LANG}}) - include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/") - - file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC - lib/${LANG}/*.cc - ${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/*.cc) - add_library(${LOCAL_SCHEDULER_LIBRARY_LANG} SHARED - ${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC}) - - if(APPLE) - if ("${LANG}" STREQUAL "python") - SET_TARGET_PROPERTIES(${LOCAL_SCHEDULER_LIBRARY_LANG} PROPERTIES SUFFIX .so) - endif() - target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) - else(APPLE) - target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) - endif(APPLE) - - add_dependencies(${LOCAL_SCHEDULER_LIBRARY_LANG} gen_local_scheduler_fbs) - - install(TARGETS ${LOCAL_SCHEDULER_LIBRARY_LANG} DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler) -endmacro() - -if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - set_local_scheduler_library("python") -endif() - -if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") - add_compile_options("-I$ENV{JAVA_HOME}/include/") - if(WIN32) - add_compile_options("-I$ENV{JAVA_HOME}/include/win32") - elseif(APPLE) - add_compile_options("-I$ENV{JAVA_HOME}/include/darwin") - else() # linux - add_compile_options("-I$ENV{JAVA_HOME}/include/linux") - endif() - set_local_scheduler_library("java") -endif() diff --git a/src/local_scheduler/build/.gitkeep b/src/local_scheduler/build/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/local_scheduler/format/local_scheduler.fbs b/src/local_scheduler/format/local_scheduler.fbs deleted file mode 100644 index a23bb28f0..000000000 --- a/src/local_scheduler/format/local_scheduler.fbs +++ /dev/null @@ -1,130 +0,0 @@ -// Local scheduler protocol specification -namespace ray.local_scheduler.protocol; - -enum MessageType:int { - // Task is submitted to the local scheduler. This is sent from a worker to a - // local scheduler. - SubmitTask = 1, - // Notify the local scheduler that a task has finished. This is sent from a - // worker to a local scheduler. - TaskDone, - // Log a message to the event table. This is sent from a worker to a local - // scheduler. - EventLogMessage, - // Send an initial connection message to the local scheduler. This is sent - // from a worker or driver to a local scheduler. - RegisterClientRequest, - // Send a reply confirming the successful registration of a worker or driver. - // This is sent from the local scheduler to a worker or driver. - RegisterClientReply, - // Notify the local scheduler that this client disconnected unexpectedly. - // This is sent from a worker to a local scheduler. - DisconnectClient, - // Notify the local scheduler that this client is disconnecting gracefully. - // This is sent from a worker to a local scheduler. - IntentionalDisconnectClient, - // Get a new task from the local scheduler. This is sent from a worker to a - // local scheduler. - GetTask, - // Tell a worker to execute a task. This is sent from a local scheduler to a - // worker. - ExecuteTask, - // Reconstruct or fetch possibly lost objects. This is sent from a worker to - // a local scheduler. - ReconstructObjects, - // For a worker that was blocked on some object(s), tell the local scheduler - // that the worker is now unblocked. This is sent from a worker to a local - // scheduler. - NotifyUnblocked, - // Add a result table entry for an object put. - PutObject, - // A request to get the task frontier for an actor, called by the actor when - // saving a checkpoint. - GetActorFrontierRequest, - // The ActorFrontier response to a GetActorFrontierRequest. The local - // scheduler returns the actor's per-handle task counts and execution - // dependencies, which can later be used as the argument to SetActorFrontier - // when resuming from the checkpoint. - GetActorFrontierReply, - // A request to set the task frontier for an actor, called when resuming from - // a checkpoint. The local scheduler will update the actor's per-handle task - // counts and execution dependencies, discard any tasks that already executed - // before the checkpoint, and make any tasks on the frontier runnable by - // making their execution dependencies available. - SetActorFrontier -} - -table SubmitTaskRequest { - execution_dependencies: [string]; - task_spec: string; -} - -// This message is sent from the local scheduler to a worker. -table GetTaskReply { - // A string of bytes representing the task specification. - task_spec: string; - // The IDs of the GPUs that the worker is allowed to use for this task. - gpu_ids: [int]; -} - -table EventLogMessage { - key: string; - value: string; - timestamp: double; -} - -// This struct is used to register a new worker with the local scheduler. -// It is shipped as part of local_scheduler_connect. -table RegisterClientRequest { - // True if the client is a worker and false if the client is a driver. - is_worker: bool; - // The ID of the worker or driver. - client_id: string; - // The process ID of this worker. - worker_pid: long; - // The driver ID. This is non-nil if the client is a driver. - driver_id: string; -} - -table DisconnectClient { -} - -table ReconstructObjects { - // List of object IDs of the objects that we want to reconstruct or fetch. - object_ids: [string]; - // Do we only want to fetch the objects or also reconstruct them? - fetch_only: bool; -} - -table PutObject { - // Task ID of the task that performed the put. - task_id: string; - // Object ID of the object that is being put. - object_id: string; -} - -// The ActorFrontier is used to represent the current frontier of tasks that -// the local scheduler has marked as runnable for a particular actor. It is -// used to save the point in an actor's lifetime at which a checkpoint was -// taken, so that the same frontier of tasks can be made runnable again if the -// actor is resumed from that checkpoint. -table ActorFrontier { - // Actor ID of the actor whose frontier is described. - actor_id: string; - // A list of handle IDs, representing the callers of the actor that have - // submitted a runnable task to the local scheduler. A nil ID represents the - // creator of the actor. - handle_ids: [string]; - // A list representing the number of tasks executed so far, per handle. Each - // count in task_counters corresponds to the handle at the same in index in - // handle_ids. - task_counters: [long]; - // A list representing the execution dependency for the next runnable task, - // per handle. Each execution dependency in frontier_dependencies corresponds - // to the handle at the same in index in handle_ids. - frontier_dependencies: [string]; -} - -table GetActorFrontierRequest { - actor_id: string; -} diff --git a/src/local_scheduler/local_scheduler.cc b/src/local_scheduler/local_scheduler.cc deleted file mode 100644 index d2c50c3fb..000000000 --- a/src/local_scheduler/local_scheduler.cc +++ /dev/null @@ -1,1555 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "common.h" -#include "common_protocol.h" -#include "event_loop.h" -#include "format/local_scheduler_generated.h" -#include "io.h" -#include "local_scheduler.h" -#include "local_scheduler_algorithm.h" -#include "local_scheduler_shared.h" -#include "logging.h" -#include "net.h" -#include "ray/util/util.h" -#include "state/actor_notification_table.h" -#include "state/db.h" -#include "state/db_client_table.h" -#include "state/driver_table.h" -#include "state/error_table.h" -#include "state/object_table.h" -#include "state/task_table.h" - -using MessageType = ray::local_scheduler::protocol::MessageType; - -/** - * A helper function for printing available and requested resource information. - * - * @param state Local scheduler state. - * @param spec Task specification object. - * @return Void. - */ -void print_resource_info(const LocalSchedulerState *state, - const TaskSpec *spec) { -#if RAY_COMMON_LOG_LEVEL <= RAY_COMMON_DEBUG - // Print information about available and requested resources. - std::cout << "Static Resources: " << std::endl; - for (auto const &resource_pair : state->static_resources) { - std::cout << " " << resource_pair.first << ": " << resource_pair.second - << std::endl; - } - std::cout << "Dynamic Resources: " << std::endl; - for (auto const &resource_pair : state->dynamic_resources) { - std::cout << " " << resource_pair.first << ": " << resource_pair.second - << std::endl; - } - if (spec) { - std::cout << "Task Required Resources: " << std::endl; - for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) { - std::cout << " " << resource_pair.first << ": " << resource_pair.second - << std::endl; - } - } -#endif -} - -int force_kill_worker(event_loop *loop, timer_id id, void *context) { - LocalSchedulerClient *worker = (LocalSchedulerClient *) context; - kill(worker->pid, SIGKILL); - close(worker->sock); - delete worker; - return EVENT_LOOP_TIMER_DONE; -} - -void kill_worker(LocalSchedulerState *state, - LocalSchedulerClient *worker, - bool cleanup, - bool suppress_warning) { - /* Erase the local scheduler's reference to the worker. */ - auto it = std::find(state->workers.begin(), state->workers.end(), worker); - RAY_CHECK(it != state->workers.end()); - state->workers.erase(it); - - /* Make sure that we removed the worker. */ - it = std::find(state->workers.begin(), state->workers.end(), worker); - RAY_CHECK(it == state->workers.end()); - - /* Release any resources held by the worker. It's important to do this before - * calling handle_worker_removed and handle_actor_worker_disconnect because - * freeing up resources here will allow the scheduling algorithm to dispatch - * more tasks. */ - release_resources(state, worker, worker->resources_in_use); - - /* Erase the algorithm state's reference to the worker. */ - if (worker->actor_id.is_nil()) { - handle_worker_removed(state, state->algorithm_state, worker); - } else { - /* Let the scheduling algorithm process the absence of this worker. */ - handle_actor_worker_disconnect(state, state->algorithm_state, worker, - cleanup); - } - - /* Remove the client socket from the event loop so that we don't process the - * SIGPIPE when the worker is killed. */ - event_loop_remove_file(state->loop, worker->sock); - - /* If the worker has registered a process ID with us and it's a child - * process, use it to send a kill signal. */ - bool free_worker = true; - if (worker->is_child && worker->pid != 0) { - /* If worker is a driver, we should not enter this condition because - * worker->pid should be 0. */ - if (cleanup) { - /* If we're exiting the local scheduler anyway, it's okay to force kill - * the worker immediately. Wait for the process to exit. */ - kill(worker->pid, SIGKILL); - waitpid(worker->pid, NULL, 0); - close(worker->sock); - } else { - /* If we're just cleaning up a single worker, allow it some time to clean - * up its state before force killing. The client socket will be closed - * and the worker struct will be freed after the timeout. */ - kill(worker->pid, SIGTERM); - event_loop_add_timer( - state->loop, RayConfig::instance().kill_worker_timeout_milliseconds(), - force_kill_worker, (void *) worker); - free_worker = false; - } - RAY_LOG(DEBUG) << "Killed worker with pid " << worker->pid; - } - - /* If this worker is still running a task and we aren't cleaning up, push an - * error message to the driver responsible for the task. */ - if (worker->task_in_progress != NULL && !cleanup && !suppress_warning) { - TaskSpec *spec = Task_task_execution_spec(worker->task_in_progress)->Spec(); - - std::ostringstream error_message; - error_message << "The worker with ID " << worker->client_id << " died or " - << "was killed while executing the task with ID " - << TaskSpec_task_id(spec); - push_error(state->db, TaskSpec_driver_id(spec), ErrorIndex::WORKER_DIED, - error_message.str()); - } - - /* Clean up the task in progress. */ - if (worker->task_in_progress) { - /* Update the task table to reflect that the task failed to complete. */ - if (state->db != NULL) { - Task_set_state(worker->task_in_progress, TaskStatus::LOST); - task_table_update(state->db, worker->task_in_progress, NULL, NULL, NULL); - } else { - Task_free(worker->task_in_progress); - } - } - - RAY_LOG(DEBUG) << "Killed worker with pid " << worker->pid; - if (free_worker) { - /* Clean up the client socket after killing the worker so that the worker - * can't receive the SIGPIPE before exiting. */ - close(worker->sock); - delete worker; - } -} - -void LocalSchedulerState_free(LocalSchedulerState *state) { - /* Reset the SIGTERM handler to default behavior, so we try to clean up the - * local scheduler at most once. If a SIGTERM is caught afterwards, there is - * the possibility of orphan worker processes. */ - signal(SIGTERM, SIG_DFL); - /* Send a null heartbeat that tells the global scheduler that we are dead to - * avoid waiting for the heartbeat timeout. */ - if (state->db != NULL) { - local_scheduler_table_disconnect(state->db); - } - - /* Kill any child processes that didn't register as a worker yet. */ - for (auto const &worker_pid : state->child_pids) { - kill(worker_pid, SIGKILL); - waitpid(worker_pid, NULL, 0); - RAY_LOG(INFO) << "Killed worker pid " << worker_pid - << " which hadn't started yet."; - } - - /* Kill any registered workers. */ - /* TODO(swang): It's possible that the local scheduler will exit before all - * of its task table updates make it to redis. */ - while (state->workers.size() > 0) { - /* Note that kill_worker modifies the container state->workers, so it is - * important to do this loop in a way that does not use invalidated - * iterators. */ - kill_worker(state, state->workers.back(), true, false); - } - - /* Disconnect from plasma. */ - ARROW_CHECK_OK(state->plasma_conn->Disconnect()); - delete state->plasma_conn; - state->plasma_conn = NULL; - - /* Clean up the database connection. NOTE(swang): The global scheduler is - * responsible for deleting our entry from the db_client table, so do not - * delete it here. */ - if (state->db != NULL) { - DBHandle_free(state->db); - } - - /* Free the command for starting new workers. */ - if (state->config.start_worker_command != NULL) { - int i = 0; - const char *arg = state->config.start_worker_command[i]; - while (arg != NULL) { - free((void *) arg); - ++i; - arg = state->config.start_worker_command[i]; - } - free(state->config.start_worker_command); - state->config.start_worker_command = NULL; - } - - /* Free the algorithm state. */ - SchedulingAlgorithmState_free(state->algorithm_state); - state->algorithm_state = NULL; - - event_loop *loop = state->loop; - - /* Free the scheduler state. */ - delete state; - - /* Destroy the event loop. */ - destroy_outstanding_callbacks(loop); - event_loop_destroy(loop); -} - -void start_worker(LocalSchedulerState *state) { - /* We can't start a worker if we don't have the path to the worker script. */ - if (state->config.start_worker_command == NULL) { - RAY_LOG(DEBUG) << "No valid command to start worker provided. Cannot start " - << "worker."; - return; - } - /* Launch the process to create the worker. */ - pid_t pid = fork(); - if (pid != 0) { - state->child_pids.push_back(pid); - RAY_LOG(DEBUG) << "Started worker with pid " << pid; - return; - } - - /* Reset the SIGCHLD handler so that it doesn't influence the worker. */ - signal(SIGCHLD, SIG_DFL); - - std::vector command_vector; - for (int i = 0; state->config.start_worker_command[i] != NULL; i++) { - command_vector.push_back(state->config.start_worker_command[i]); - } - - /* Add a NULL pointer to the end. */ - command_vector.push_back(NULL); - - /* Try to execute the worker command. Exit if we're not successful. */ - execvp(command_vector[0], (char *const *) command_vector.data()); - - LocalSchedulerState_free(state); - RAY_LOG(FATAL) << "Failed to start worker"; -} - -/** - * Parse the command to start a worker. This takes in the command string, - * splits it into tokens on the space characters, and allocates an array of the - * tokens, terminated by a NULL pointer. - * - * @param command The command string to start a worker. - * @return A pointer to an array of strings, the tokens in the command string. - * The last element is a NULL pointer. - */ -const char **parse_command(const char *command) { - /* Count the number of tokens. */ - char *command_copy = strdup(command); - const char *delimiter = " "; - char *token = NULL; - int num_args = 0; - token = strtok(command_copy, delimiter); - while (token != NULL) { - ++num_args; - token = strtok(NULL, delimiter); - } - free(command_copy); - - /* Allocate a NULL-terminated array for the tokens. */ - const char **command_args = - (const char **) malloc((num_args + 1) * sizeof(const char *)); - command_args[num_args] = NULL; - - /* Fill in the token array. */ - command_copy = strdup(command); - token = strtok(command_copy, delimiter); - int i = 0; - while (token != NULL) { - command_args[i] = strdup(token); - ++i; - token = strtok(NULL, delimiter); - } - free(command_copy); - - RAY_CHECK(num_args == i); - return command_args; -} - -LocalSchedulerState *LocalSchedulerState_init( - const char *node_ip_address, - event_loop *loop, - const char *redis_primary_addr, - int redis_primary_port, - const char *local_scheduler_socket_name, - const char *plasma_store_socket_name, - const char *plasma_manager_socket_name, - const char *plasma_manager_address, - bool global_scheduler_exists, - const std::unordered_map &static_resource_conf, - const char *start_worker_command, - int num_workers) { - LocalSchedulerState *state = new LocalSchedulerState(); - /* Set the configuration struct for the local scheduler. */ - if (start_worker_command != NULL) { - state->config.start_worker_command = parse_command(start_worker_command); - } else { - state->config.start_worker_command = NULL; - } - if (start_worker_command == NULL) { - RAY_LOG(WARNING) << "No valid command to start a worker provided, local " - << "scheduler will not start any workers."; - } - state->config.global_scheduler_exists = global_scheduler_exists; - - state->loop = loop; - - /* Connect to Redis if a Redis address is provided. */ - if (redis_primary_addr != NULL) { - /* Construct db_connect_args */ - std::vector db_connect_args; - db_connect_args.push_back("local_scheduler_socket_name"); - db_connect_args.push_back(local_scheduler_socket_name); - for (auto const &resource_pair : static_resource_conf) { - // TODO(rkn): This could cause issues if a resource name collides with - // another field name "manager_address". - db_connect_args.push_back(resource_pair.first); - db_connect_args.push_back(std::to_string(resource_pair.second)); - } - - if (plasma_manager_address != NULL) { - db_connect_args.push_back("manager_address"); - db_connect_args.push_back(plasma_manager_address); - } - - state->db = db_connect(std::string(redis_primary_addr), redis_primary_port, - "local_scheduler", node_ip_address, db_connect_args); - db_attach(state->db, loop, false); - } else { - state->db = NULL; - } - /* Connect to Plasma. This method will retry if Plasma hasn't started yet. */ - state->plasma_conn = new plasma::PlasmaClient(); - if (plasma_manager_socket_name != NULL) { - ARROW_CHECK_OK(state->plasma_conn->Connect( - plasma_store_socket_name, plasma_manager_socket_name, - plasma::kPlasmaDefaultReleaseDelay)); - } else { - ARROW_CHECK_OK(state->plasma_conn->Connect( - plasma_store_socket_name, "", plasma::kPlasmaDefaultReleaseDelay)); - } - /* Subscribe to notifications about sealed objects. */ - int plasma_fd; - ARROW_CHECK_OK(state->plasma_conn->Subscribe(&plasma_fd)); - /* Add the callback that processes the notification to the event loop. */ - event_loop_add_file(loop, plasma_fd, EVENT_LOOP_READ, - process_plasma_notification, state); - /* Add scheduler state. */ - state->algorithm_state = SchedulingAlgorithmState_init(); - - /* Initialize resource vectors. */ - state->static_resources = static_resource_conf; - state->dynamic_resources = static_resource_conf; - /* Initialize available GPUs. */ - if (state->static_resources.count("GPU") == 1) { - for (int i = 0; i < state->static_resources["GPU"]; ++i) { - state->available_gpus.push_back(i); - } - } - /* Print some debug information about resource configuration. */ - print_resource_info(state, NULL); - - /* Start the initial set of workers. */ - for (int i = 0; i < num_workers; ++i) { - start_worker(state); - } - - /* Initialize the time at which the previous heartbeat was sent. */ - state->previous_heartbeat_time = current_time_ms(); - - return state; -} - -/* TODO(atumanov): vectorize resource counts on input. */ -bool check_dynamic_resources( - LocalSchedulerState *state, - const std::unordered_map &resources) { - for (auto const &resource_pair : resources) { - std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - if (state->dynamic_resources[resource_name] < resource_quantity) { - return false; - } - } - return true; -} - -void resource_sanity_checks(LocalSchedulerState *state, - LocalSchedulerClient *worker) { - // Check the resources in use by the worker. - for (auto const &resource_pair : worker->resources_in_use) { - const std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - - RAY_CHECK(state->dynamic_resources[resource_name] <= - state->static_resources[resource_name]); - if (resource_name != std::string("CPU")) { - RAY_CHECK(state->dynamic_resources[resource_name] >= 0); - } - - RAY_CHECK(resource_quantity >= 0); - RAY_CHECK(resource_quantity <= state->static_resources[resource_name]); - } -} - -/* TODO(atumanov): just pass the required resource vector of doubles. */ -void acquire_resources( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - const std::unordered_map &resources) { - // Loop over each required resource type and acquire the appropriate quantity. - for (auto const &resource_pair : resources) { - const std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - - // Do some special handling for GPU resources. - if (resource_name == std::string("GPU")) { - if (resource_quantity != 0) { - // Make sure that the worker isn't using any GPUs already. - RAY_CHECK(worker->gpus_in_use.size() == 0); - RAY_CHECK(state->available_gpus.size() >= resource_quantity); - // Reserve GPUs for the worker. - for (int i = 0; i < resource_quantity; i++) { - worker->gpus_in_use.push_back(state->available_gpus.back()); - state->available_gpus.pop_back(); - } - } - } - - // Do bookkeeping for general resource types. - if (resource_name != std::string("CPU")) { - RAY_CHECK(state->dynamic_resources[resource_name] >= resource_quantity); - } - state->dynamic_resources[resource_name] -= resource_quantity; - worker->resources_in_use[resource_name] += resource_quantity; - } - - // Do some sanity checks. - resource_sanity_checks(state, worker); -} - -void release_resources( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - const std::unordered_map &resources) { - for (auto const &resource_pair : resources) { - const std::string resource_name = resource_pair.first; - double resource_quantity = resource_pair.second; - - // Do some special handling for GPU resources. - if (resource_name == std::string("GPU")) { - if (resource_quantity != 0) { - RAY_CHECK(resource_quantity == worker->gpus_in_use.size()); - // Move the GPU IDs the worker was using back to the local scheduler. - for (auto const &gpu_id : worker->gpus_in_use) { - state->available_gpus.push_back(gpu_id); - } - worker->gpus_in_use.clear(); - } - } - - // Do bookkeeping for general resources types. - state->dynamic_resources[resource_name] += resource_quantity; - worker->resources_in_use[resource_name] -= resource_quantity; - } - - // Do some sanity checks. - resource_sanity_checks(state, worker); -} - -bool is_driver_alive(LocalSchedulerState *state, WorkerID driver_id) { - return state->removed_drivers.count(driver_id) == 0; -} - -void assign_task_to_worker(LocalSchedulerState *state, - TaskExecutionSpec &execution_spec, - LocalSchedulerClient *worker) { - int64_t task_spec_size = execution_spec.SpecSize(); - TaskSpec *spec = execution_spec.Spec(); - // Acquire the necessary resources for running this task. - const std::unordered_map required_resources = - TaskSpec_get_required_resources(spec); - acquire_resources(state, worker, required_resources); - // Check that actor tasks don't have non-CPU requirements. Any necessary - // non-CPU resources (in particular, GPUs) should already have been acquired - // by the actor worker. - if (!worker->actor_id.is_nil()) { - RAY_CHECK(required_resources.size() == 1); - RAY_CHECK(required_resources.count("CPU") == 1); - } - - RAY_CHECK(worker->actor_id == TaskSpec_actor_id(spec)); - /* Make sure the driver for this task is still alive. */ - WorkerID driver_id = TaskSpec_driver_id(spec); - RAY_CHECK(is_driver_alive(state, driver_id)); - - /* Construct a flatbuffer object to send to the worker. */ - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::local_scheduler::protocol::CreateGetTaskReply( - fbb, fbb.CreateString((char *) spec, task_spec_size), - fbb.CreateVector(worker->gpus_in_use)); - fbb.Finish(message); - - if (write_message(worker->sock, - static_cast(MessageType::ExecuteTask), - fbb.GetSize(), (uint8_t *) fbb.GetBufferPointer()) < 0) { - if (errno == EPIPE || errno == EBADF) { - /* Something went wrong, so kill the worker. */ - kill_worker(state, worker, false, false); - RAY_LOG(WARNING) << "Failed to give task to worker on fd " << worker->sock - << ". The client may have hung up."; - } else { - RAY_LOG(FATAL) << "Failed to give task to client on fd " << worker->sock; - } - } - - Task *task = - Task_alloc(execution_spec, TaskStatus::RUNNING, - state->db ? get_db_client_id(state->db) : DBClientID::nil()); - /* Record which task this worker is executing. This will be freed in - * process_message when the worker sends a GetTask message to the local - * scheduler. */ - worker->task_in_progress = Task_copy(task); - /* Update the global task table. */ - if (state->db != NULL) { - task_table_update(state->db, task, NULL, NULL, NULL); - } else { - Task_free(task); - } -} - -// This is used to allow task_table_update to fail. -void allow_task_table_update_failure(UniqueID id, - void *user_context, - void *user_data) {} - -void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker) { - if (worker->task_in_progress != NULL) { - TaskSpec *spec = Task_task_execution_spec(worker->task_in_progress)->Spec(); - // Return dynamic resources back for the task in progress. - if (TaskSpec_is_actor_creation_task(spec)) { - // Resources required by the actor creation task are acquired for the - // actor's lifetime, so don't return anything here. TODO(rkn): Should the - // actor creation task require 1 CPU in addition to any resources acquired - // for the lifetime of the actor? If not, then the local scheduler may - // schedule an arbitrary number of actor creation tasks concurrently (if - // they don't acquire any resources for their entire lifetime). In - // practice this will usually be rate-limited by the rate at which we can - // create new workers. - - ActorID actor_creation_id = TaskSpec_actor_creation_id(spec); - WorkerID driver_id = TaskSpec_driver_id(spec); - - // The driver must be alive because if the driver had been removed, then - // this worker would have been killed (because it was executing a task for - // the driver). - RAY_CHECK(is_driver_alive(state, driver_id)); - - // Update the worker struct with this actor ID. - RAY_CHECK(worker->actor_id.is_nil()); - worker->actor_id = actor_creation_id; - // Extract the initial execution dependency from the actor creation task. - RAY_CHECK(TaskSpec_num_returns(spec) == 1); - ObjectID initial_execution_dependency = TaskSpec_return(spec, 0); - // Let the scheduling algorithm process the presence of this new worker. - handle_convert_worker_to_actor(state, state->algorithm_state, - actor_creation_id, - initial_execution_dependency, worker); - // Publish the actor creation notification. The corresponding callback - // handle_actor_creation_callback will update state->actor_mapping. - publish_actor_creation_notification( - state->db, actor_creation_id, driver_id, get_db_client_id(state->db)); - } else if (worker->actor_id.is_nil()) { - // Return dynamic resources back for the task in progress. - RAY_CHECK(worker->resources_in_use["CPU"] == - TaskSpec_get_required_resource(spec, "CPU")); - // Return GPU resources. - RAY_CHECK(worker->gpus_in_use.size() == - TaskSpec_get_required_resource(spec, "GPU")); - release_resources(state, worker, worker->resources_in_use); - } else { - // Actor tasks should only specify CPU requirements. - RAY_CHECK(0 == TaskSpec_get_required_resource(spec, "GPU")); - std::unordered_map cpu_resources; - cpu_resources["CPU"] = TaskSpec_get_required_resource(spec, "CPU"); - release_resources(state, worker, cpu_resources); - } - /* If we're connected to Redis, update tables. */ - if (state->db != NULL) { - /* Update control state tables. */ - TaskStatus task_state = TaskStatus::DONE; - Task_set_state(worker->task_in_progress, task_state); - auto retryInfo = RetryInfo{ - .num_retries = 0, // This value is unused. - .timeout = 0, // This value is unused. - .fail_callback = allow_task_table_update_failure, - }; - - // We allow this call to fail in case the driver has been removed and the - // task table entries have already been cleaned up by the monitor. - task_table_update(state->db, worker->task_in_progress, &retryInfo, NULL, - NULL); - } else { - Task_free(worker->task_in_progress); - } - /* The call to task_table_update takes ownership of the - * task_in_progress, so we set the pointer to NULL so it is not used. */ - worker->task_in_progress = NULL; - } -} - -void process_plasma_notification(event_loop *loop, - int client_sock, - void *context, - int events) { - LocalSchedulerState *state = (LocalSchedulerState *) context; - /* Read the notification from Plasma. */ - uint8_t *notification = read_message_async(loop, client_sock); - if (!notification) { - /* The store has closed the socket. */ - LocalSchedulerState_free(state); - RAY_LOG(FATAL) << "Lost connection to the plasma store, local scheduler is " - << "exiting!"; - } - auto object_info = flatbuffers::GetRoot(notification); - ObjectID object_id = from_flatbuf(*object_info->object_id()); - if (object_info->is_deletion()) { - handle_object_removed(state, object_id); - } else { - handle_object_available(state, state->algorithm_state, object_id); - } - free(notification); -} - -void reconstruct_task_update_callback(Task *task, - void *user_context, - bool updated) { - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - if (!updated) { - /* The test-and-set failed. The task is either: (1) not finished yet, (2) - * lost, but not yet updated, or (3) already being reconstructed. */ - DBClientID current_local_scheduler_id = Task_local_scheduler(task); - if (!current_local_scheduler_id.is_nil()) { - DBClient current_local_scheduler = - db_client_table_cache_get(state->db, current_local_scheduler_id); - if (!current_local_scheduler.is_alive) { - /* (2) The current local scheduler for the task is dead. The task is - * lost, but the task table hasn't received the update yet. Retry the - * test-and-set. */ - task_table_test_and_update(state->db, Task_task_id(task), - current_local_scheduler_id, Task_state(task), - TaskStatus::RECONSTRUCTING, NULL, - reconstruct_task_update_callback, state); - } - } - /* The test-and-set failed, so it is not safe to resubmit the task for - * execution. Suppress the request. */ - return; - } - - /* Otherwise, the test-and-set succeeded, so resubmit the task for execution - * to ensure that reconstruction will happen. */ - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - TaskSpec *spec = execution_spec->Spec(); - if (TaskSpec_actor_id(spec).is_nil()) { - handle_task_submitted(state, state->algorithm_state, *execution_spec); - } else { - handle_actor_task_submitted(state, state->algorithm_state, *execution_spec); - } - - /* Recursively reconstruct the task's inputs, if necessary. */ - int64_t num_dependencies = execution_spec->NumDependencies(); - for (int64_t i = 0; i < num_dependencies; ++i) { - int count = execution_spec->DependencyIdCount(i); - for (int64_t j = 0; j < count; ++j) { - ObjectID dependency_id = execution_spec->DependencyId(i, j); - reconstruct_object(state, dependency_id); - } - } -} - -void reconstruct_put_task_update_callback(Task *task, - void *user_context, - bool updated) { - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - if (!updated) { - /* The test-and-set failed. The task is either: (1) not finished yet, (2) - * lost, but not yet updated, or (3) already being reconstructed. */ - DBClientID current_local_scheduler_id = Task_local_scheduler(task); - if (!current_local_scheduler_id.is_nil()) { - DBClient current_local_scheduler = - db_client_table_cache_get(state->db, current_local_scheduler_id); - if (!current_local_scheduler.is_alive) { - /* (2) The current local scheduler for the task is dead. The task is - * lost, but the task table hasn't received the update yet. Retry the - * test-and-set. */ - task_table_test_and_update(state->db, Task_task_id(task), - current_local_scheduler_id, Task_state(task), - TaskStatus::RECONSTRUCTING, NULL, - reconstruct_put_task_update_callback, state); - } else if (Task_state(task) == TaskStatus::RUNNING) { - /* (1) The task is still executing on a live node. The object created - * by `ray.put` was not able to be reconstructed, and the workload will - * likely hang. Push an error to the appropriate driver. */ - TaskSpec *spec = Task_task_execution_spec(task)->Spec(); - - std::ostringstream error_message; - error_message << "The task with ID " << TaskSpec_task_id(spec) - << " is still executing and so the object created by " - << "ray.put could not be reconstructed."; - push_error(state->db, TaskSpec_driver_id(spec), - ErrorIndex::PUT_RECONSTRUCTION, error_message.str()); - } - } else { - /* (1) The task is still executing and it is the driver task. We cannot - * restart the driver task, so the workload will hang. Push an error to - * the appropriate driver. */ - TaskSpec *spec = Task_task_execution_spec(task)->Spec(); - - std::ostringstream error_message; - error_message << "The task with ID " << TaskSpec_task_id(spec) - << " is a driver task and so the object created by ray.put " - << "could not be reconstructed."; - push_error(state->db, TaskSpec_driver_id(spec), - ErrorIndex::PUT_RECONSTRUCTION, error_message.str()); - } - } else { - /* The update to TaskStatus::RECONSTRUCTING succeeded, so continue with - * reconstruction as usual. */ - reconstruct_task_update_callback(task, user_context, updated); - } -} - -void reconstruct_evicted_result_lookup_callback(ObjectID reconstruct_object_id, - TaskID task_id, - bool is_put, - void *user_context) { - RAY_CHECK(!task_id.is_nil()) - << "No task information found for object during reconstruction"; - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - - task_table_test_and_update_callback done_callback; - if (is_put) { - /* If the evicted object was created through ray.put and the originating - * task - * is still executing, it's very likely that the workload will hang and the - * worker needs to be restarted. Else, the reconstruction behavior is the - * same as for other evicted objects */ - done_callback = reconstruct_put_task_update_callback; - } else { - done_callback = reconstruct_task_update_callback; - } - /* If there are no other instances of the task running, it's safe for us to - * claim responsibility for reconstruction. */ - task_table_test_and_update(state->db, task_id, DBClientID::nil(), - (TaskStatus::DONE | TaskStatus::LOST), - TaskStatus::RECONSTRUCTING, NULL, done_callback, - state); -} - -void reconstruct_failed_result_lookup_callback(ObjectID reconstruct_object_id, - TaskID task_id, - bool is_put, - void *user_context) { - if (task_id.is_nil()) { - /* NOTE(swang): For some reason, the result table update sometimes happens - * after this lookup returns, possibly due to concurrent clients. In most - * cases, this is okay because the initial execution is probably still - * pending, so for now, we log a warning and suppress reconstruction. */ - RAY_LOG(WARNING) << "No task information found for object during " - << "reconstruction (no object entry yet)"; - return; - } - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - /* If the task failed to finish, it's safe for us to claim responsibility for - * reconstruction. */ - task_table_test_and_update(state->db, task_id, DBClientID::nil(), - TaskStatus::LOST, TaskStatus::RECONSTRUCTING, NULL, - reconstruct_task_update_callback, state); -} - -void reconstruct_object_lookup_callback( - ObjectID reconstruct_object_id, - bool never_created, - const std::vector &manager_ids, - void *user_context) { - RAY_LOG(DEBUG) << "Manager count was " << manager_ids.size(); - /* Only continue reconstruction if we find that the object doesn't exist on - * any nodes. NOTE: This codepath is not responsible for checking if the - * object table entry is up-to-date. */ - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - /* Look up the task that created the object in the result table. */ - if (never_created) { - /* If the object has not been created yet, we reconstruct the object if and - * only if the task that created the object failed to complete. */ - result_table_lookup(state->db, reconstruct_object_id, NULL, - reconstruct_failed_result_lookup_callback, - (void *) state); - } else { - /* If the object has been created, filter out the dead plasma managers that - * have it. */ - size_t num_live_managers = 0; - for (auto manager_id : manager_ids) { - DBClient manager = db_client_table_cache_get(state->db, manager_id); - if (manager.is_alive) { - num_live_managers++; - } - } - /* If the object was created, but all plasma managers that had the object - * either evicted it or failed, we reconstruct the object if and only if - * there are no other instances of the task running. */ - if (num_live_managers == 0) { - result_table_lookup(state->db, reconstruct_object_id, NULL, - reconstruct_evicted_result_lookup_callback, - (void *) state); - } - } -} - -void reconstruct_object(LocalSchedulerState *state, - ObjectID reconstruct_object_id) { - RAY_LOG(DEBUG) << "Starting reconstruction"; - /* If the object is locally available, no need to reconstruct. */ - if (object_locally_available(state->algorithm_state, reconstruct_object_id)) { - return; - } - /* Determine if reconstruction is necessary by checking if the object exists - * on a node. */ - RAY_CHECK(state->db != NULL); - object_table_lookup(state->db, reconstruct_object_id, NULL, - reconstruct_object_lookup_callback, (void *) state); -} - -void handle_client_register( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - const ray::local_scheduler::protocol::RegisterClientRequest *message) { - /* Make sure this worker hasn't already registered. */ - RAY_CHECK(!worker->registered); - worker->registered = true; - worker->is_worker = message->is_worker(); - RAY_CHECK(worker->client_id.is_nil()); - worker->client_id = from_flatbuf(*message->client_id()); - - /* Register the worker or driver. */ - if (worker->is_worker) { - /* Update the actor mapping with the actor ID of the worker (if an actor is - * running on the worker). */ - worker->pid = message->worker_pid(); - /* Register worker process id with the scheduler. */ - /* Determine if this worker is one of our child processes. */ - RAY_LOG(DEBUG) << "PID is " << worker->pid; - auto it = std::find(state->child_pids.begin(), state->child_pids.end(), - worker->pid); - if (it != state->child_pids.end()) { - /* If this worker is one of our child processes, mark it as a child so - * that we know that we can wait for the process to exit during - * cleanup. */ - worker->is_child = true; - state->child_pids.erase(it); - RAY_LOG(DEBUG) << "Found matching child pid " << worker->pid; - } - } else { - /* Register the driver. Currently we don't do anything here. */ - } -} - -void handle_driver_removed_callback(WorkerID driver_id, void *user_context) { - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - - /* Kill any actors that were created by the removed driver, and kill any - * workers that are currently running tasks from the dead driver. */ - auto it = state->workers.begin(); - while (it != state->workers.end()) { - /* Increment the iterator by one before calling kill_worker, because - * kill_worker will invalidate the iterator. Note that this requires - * knowledge of the particular container that we are iterating over (in this - * case it is a list). */ - auto next_it = it; - next_it++; - - ActorID actor_id = (*it)->actor_id; - Task *task = (*it)->task_in_progress; - - if (!actor_id.is_nil()) { - /* This is an actor. */ - RAY_CHECK(state->actor_mapping.count(actor_id) == 1); - if (state->actor_mapping[actor_id].driver_id == driver_id) { - /* This actor was created by the removed driver, so kill the actor. */ - RAY_LOG(DEBUG) << "Killing an actor for a removed driver."; - kill_worker(state, *it, false, true); - } - } else if (task != NULL) { - TaskSpec *spec = Task_task_execution_spec(task)->Spec(); - if (TaskSpec_driver_id(spec) == driver_id) { - RAY_LOG(DEBUG) << "Killing a worker executing a task for a removed " - << "driver."; - kill_worker(state, *it, false, true); - } - } - - it = next_it; - } - - /* Add the driver to a list of dead drivers. */ - state->removed_drivers.insert(driver_id); - - /* Notify the scheduling algorithm that the driver has been removed. It should - * remove tasks for that driver from its data structures. */ - handle_driver_removed(state, state->algorithm_state, driver_id); -} - -void handle_client_disconnect(LocalSchedulerState *state, - LocalSchedulerClient *worker) { - if (!worker->registered || worker->is_worker) { - } else { - /* In this case, a driver is disconecting. */ - driver_table_send_driver_death(state->db, worker->client_id, NULL); - } - /* Suppress the warning message if the worker already disconnected. */ - kill_worker(state, worker, false, worker->disconnected); -} - -void handle_get_actor_frontier(LocalSchedulerState *state, - LocalSchedulerClient *worker, - ActorID actor_id) { - auto task_counters = - get_actor_task_counters(state->algorithm_state, actor_id); - auto frontier = get_actor_frontier(state->algorithm_state, actor_id); - - /* Build the ActorFrontier flatbuffer. */ - std::vector handle_vector; - std::vector task_counter_vector; - std::vector frontier_vector; - for (auto handle : task_counters) { - handle_vector.push_back(handle.first); - task_counter_vector.push_back(handle.second); - frontier_vector.push_back(frontier[handle.first]); - } - flatbuffers::FlatBufferBuilder fbb; - auto reply = ray::local_scheduler::protocol::CreateActorFrontier( - fbb, to_flatbuf(fbb, actor_id), to_flatbuf(fbb, handle_vector), - fbb.CreateVector(task_counter_vector), to_flatbuf(fbb, frontier_vector)); - fbb.Finish(reply); - /* Respond with the built ActorFrontier. */ - if (write_message(worker->sock, - static_cast(MessageType::GetActorFrontierReply), - fbb.GetSize(), (uint8_t *) fbb.GetBufferPointer()) < 0) { - if (errno == EPIPE || errno == EBADF) { - /* Something went wrong, so kill the worker. */ - kill_worker(state, worker, false, false); - RAY_LOG(WARNING) << "Failed to return actor frontier to worker on fd " - << worker->sock << ". The client may have hung up."; - } else { - RAY_LOG(FATAL) << "Failed to give task to client on fd " << worker->sock; - } - } -} - -void handle_set_actor_frontier( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - ray::local_scheduler::protocol::ActorFrontier const &frontier) { - /* Parse the ActorFrontier flatbuffer. */ - ActorID actor_id = from_flatbuf(*frontier.actor_id()); - std::unordered_map task_counters; - std::unordered_map frontier_dependencies; - for (size_t i = 0; i < frontier.handle_ids()->size(); ++i) { - ActorID handle_id = from_flatbuf(*frontier.handle_ids()->Get(i)); - task_counters[handle_id] = frontier.task_counters()->Get(i); - frontier_dependencies[handle_id] = - from_flatbuf(*frontier.frontier_dependencies()->Get(i)); - } - /* Set the actor's frontier. */ - set_actor_task_counters(state->algorithm_state, actor_id, task_counters); - set_actor_frontier(state, state->algorithm_state, actor_id, - frontier_dependencies); -} - -void process_message(event_loop *loop, - int client_sock, - void *context, - int events) { - int64_t start_time = current_time_ms(); - - LocalSchedulerClient *worker = (LocalSchedulerClient *) context; - LocalSchedulerState *state = worker->local_scheduler_state; - - int64_t type; - read_vector(client_sock, &type, state->input_buffer); - uint8_t *input = state->input_buffer.data(); - - RAY_LOG(DEBUG) << "New event of type " << type; - - switch (type) { - case static_cast(MessageType::SubmitTask): { - auto message = - flatbuffers::GetRoot( - input); - TaskExecutionSpec execution_spec = - TaskExecutionSpec(from_flatbuf(*message->execution_dependencies()), - (TaskSpec *) message->task_spec()->data(), - message->task_spec()->size()); - /* Set the tasks's local scheduler entrypoint time. */ - execution_spec.SetLastTimeStamp(current_time_ms()); - TaskSpec *spec = execution_spec.Spec(); - /* Update the result table, which holds mappings of object ID -> ID of the - * task that created it. */ - if (state->db != NULL) { - TaskID task_id = TaskSpec_task_id(spec); - for (int64_t i = 0; i < TaskSpec_num_returns(spec); ++i) { - ObjectID return_id = TaskSpec_return(spec, i); - result_table_add(state->db, return_id, task_id, false, NULL, NULL, - NULL); - } - } - - /* Handle the task submission. */ - if (TaskSpec_actor_id(spec).is_nil()) { - handle_task_submitted(state, state->algorithm_state, execution_spec); - } else { - handle_actor_task_submitted(state, state->algorithm_state, - execution_spec); - } - } break; - case static_cast(MessageType::TaskDone): { - } break; - case static_cast(MessageType::DisconnectClient): { - finish_task(state, worker); - RAY_CHECK(!worker->disconnected); - worker->disconnected = true; - /* If the disconnected worker was not an actor, start a new worker to make - * sure there are enough workers in the pool. */ - if (worker->actor_id.is_nil()) { - start_worker(state); - } - } break; - case static_cast(MessageType::EventLogMessage): { - /* Parse the message. */ - auto message = - flatbuffers::GetRoot( - input); - if (state->db != NULL) { - RayLogger_log_event(state->db, (uint8_t *) message->key()->data(), - message->key()->size(), - (uint8_t *) message->value()->data(), - message->value()->size(), message->timestamp()); - } - } break; - case static_cast(MessageType::RegisterClientRequest): { - auto message = flatbuffers::GetRoot< - ray::local_scheduler::protocol::RegisterClientRequest>(input); - handle_client_register(state, worker, message); - } break; - case static_cast(MessageType::GetTask): { - /* If this worker reports a completed task, account for resources. */ - finish_task(state, worker); - /* Let the scheduling algorithm process the fact that there is an available - * worker. */ - if (worker->actor_id.is_nil()) { - handle_worker_available(state, state->algorithm_state, worker); - } else { - handle_actor_worker_available(state, state->algorithm_state, worker); - } - } break; - case static_cast(MessageType::ReconstructObjects): { - auto message = flatbuffers::GetRoot< - ray::local_scheduler::protocol::ReconstructObjects>(input); - RAY_CHECK(!message->fetch_only()); - if (worker->task_in_progress != NULL && !worker->is_blocked) { - /* If the worker was executing a task (i.e. non-driver) and it wasn't - * already blocked on an object that's not locally available, update its - * state to blocked. */ - worker->is_blocked = true; - // Return the CPU resources that the blocked worker was using, but not - // other resources. If the worker is an actor, this will not return the - // CPU resources that the worker has acquired for its lifetime. It will - // only return the ones associated with the current method. - TaskSpec *spec = - Task_task_execution_spec(worker->task_in_progress)->Spec(); - std::unordered_map cpu_resources; - cpu_resources["CPU"] = TaskSpec_get_required_resource(spec, "CPU"); - release_resources(state, worker, cpu_resources); - /* Let the scheduling algorithm process the fact that the worker is - * blocked. */ - if (worker->actor_id.is_nil()) { - handle_worker_blocked(state, state->algorithm_state, worker); - } else { - handle_actor_worker_blocked(state, state->algorithm_state, worker); - } - print_worker_info("Reconstructing", state->algorithm_state); - } - RAY_CHECK(message->object_ids()->size() == 1); - ObjectID object_id = from_flatbuf(*message->object_ids()->Get(0)); - reconstruct_object(state, object_id); - } break; - case static_cast(CommonMessageType::DISCONNECT_CLIENT): { - RAY_LOG(DEBUG) << "Disconnecting client on fd " << client_sock; - handle_client_disconnect(state, worker); - } break; - case static_cast(MessageType::NotifyUnblocked): { - /* TODO(rkn): A driver may call this as well, right? */ - if (worker->task_in_progress != NULL) { - /* If the worker was executing a task (i.e. non-driver), update its - * state to not blocked. */ - RAY_CHECK(worker->is_blocked); - worker->is_blocked = false; - /* Lease back the CPU resources that the blocked worker needs (note that - * it never released its GPU resources). TODO(swang): Leasing back the - * resources to blocked workers can cause us to transiently exceed the - * maximum number of resources. This could be fixed by having blocked - * workers explicitly yield and wait to be given back resources before - * continuing execution. */ - TaskSpec *spec = - Task_task_execution_spec(worker->task_in_progress)->Spec(); - std::unordered_map cpu_resources; - cpu_resources["CPU"] = TaskSpec_get_required_resource(spec, "CPU"); - acquire_resources(state, worker, cpu_resources); - /* Let the scheduling algorithm process the fact that the worker is - * unblocked. */ - if (worker->actor_id.is_nil()) { - handle_worker_unblocked(state, state->algorithm_state, worker); - } else { - handle_actor_worker_unblocked(state, state->algorithm_state, worker); - } - } - print_worker_info("Worker unblocked", state->algorithm_state); - } break; - case static_cast(MessageType::PutObject): { - auto message = - flatbuffers::GetRoot(input); - result_table_add(state->db, from_flatbuf(*message->object_id()), - from_flatbuf(*message->task_id()), true, NULL, NULL, NULL); - } break; - case static_cast(MessageType::GetActorFrontierRequest): { - auto message = flatbuffers::GetRoot< - ray::local_scheduler::protocol::GetActorFrontierRequest>(input); - ActorID actor_id = from_flatbuf(*message->actor_id()); - handle_get_actor_frontier(state, worker, actor_id); - } break; - case static_cast(MessageType::SetActorFrontier): { - auto message = - flatbuffers::GetRoot( - input); - handle_set_actor_frontier(state, worker, *message); - } break; - default: - /* This code should be unreachable. */ - RAY_CHECK(0); - } - - /* Print a warning if this method took too long. */ - int64_t end_time = current_time_ms(); - if (end_time - start_time > - RayConfig::instance().max_time_for_handler_milliseconds()) { - RAY_LOG(WARNING) << "process_message of type " << type << " took " - << end_time - start_time << " milliseconds."; - } -} - -void new_client_connection(event_loop *loop, - int listener_sock, - void *context, - int events) { - LocalSchedulerState *state = (LocalSchedulerState *) context; - int new_socket = accept_client(listener_sock); - /* Create a struct for this worker. This will be freed when we free the local - * scheduler state. */ - LocalSchedulerClient *worker = new LocalSchedulerClient(); - worker->sock = new_socket; - worker->registered = false; - worker->disconnected = false; - /* We don't know whether this is a worker or not, so just initialize is_worker - * to false. */ - worker->is_worker = true; - worker->client_id = WorkerID::nil(); - worker->task_in_progress = NULL; - worker->is_blocked = false; - worker->pid = 0; - worker->is_child = false; - worker->actor_id = ActorID::nil(); - worker->local_scheduler_state = state; - state->workers.push_back(worker); - event_loop_add_file(loop, new_socket, EVENT_LOOP_READ, process_message, - worker); - RAY_LOG(DEBUG) << "new connection with fd " << new_socket; -} - -/* We need this code so we can clean up when we get a SIGTERM signal. */ - -LocalSchedulerState *g_state = NULL; - -void signal_handler(int signal) { - RAY_LOG(DEBUG) << "Signal was " << signal; - if (signal == SIGTERM) { - /* NOTE(swang): This call removes the SIGTERM handler to ensure that we - * free the local scheduler state at most once. If another SIGTERM is - * caught during this call, there is the possibility of orphan worker - * processes. */ - if (g_state) { - LocalSchedulerState_free(g_state); - } - exit(0); - } -} - -/* End of the cleanup code. */ - -void handle_task_scheduled_callback(Task *original_task, - void *subscribe_context) { - LocalSchedulerState *state = (LocalSchedulerState *) subscribe_context; - TaskExecutionSpec *execution_spec = Task_task_execution_spec(original_task); - TaskSpec *spec = execution_spec->Spec(); - - /* Set the tasks's local scheduler entrypoint time. */ - execution_spec->SetLastTimeStamp(current_time_ms()); - - /* If the driver for this task has been removed, then don't bother telling the - * scheduling algorithm. */ - WorkerID driver_id = TaskSpec_driver_id(spec); - if (!is_driver_alive(state, driver_id)) { - RAY_LOG(DEBUG) << "Ignoring scheduled task for removed driver."; - return; - } - - if (TaskSpec_actor_id(spec).is_nil()) { - /* This task does not involve an actor. Handle it normally. */ - handle_task_scheduled(state, state->algorithm_state, *execution_spec); - } else { - /* This task involves an actor. Call the scheduling algorithm's actor - * handler. */ - handle_actor_task_scheduled(state, state->algorithm_state, *execution_spec); - } -} - -/** - * Process a notification about the creation of a new actor. Use this to update - * the mapping from actor ID to the local scheduler ID of the local scheduler - * that is responsible for the actor. If this local scheduler is responsible for - * the actor, then launch a new worker process to create that actor. - * - * @param actor_id The ID of the actor being created. - * @param local_scheduler_id The ID of the local scheduler that is responsible - * for creating the actor. - * @param context The context for this callback. - * @return Void. - */ -void handle_actor_creation_callback(const ActorID &actor_id, - const WorkerID &driver_id, - const DBClientID &local_scheduler_id, - void *context) { - LocalSchedulerState *state = (LocalSchedulerState *) context; - - /* If the driver has been removed, don't bother doing anything. */ - if (state->removed_drivers.count(driver_id) == 1) { - return; - } - - // TODO(rkn): If we do not have perfect task suppression and it is possible - // for a task to be executed simultaneously on two nodes, then we will need to - // detect and handle that case. - - if (state->actor_mapping.count(actor_id) != 0) { - // This actor already exists. - auto it = state->actor_mapping.find(actor_id); - if (it->second.local_scheduler_id == get_db_client_id(state->db)) { - // TODO(rkn): The actor was previously assigned to this local scheduler. - // We should kill the actor here if it is still around. Also, if it hasn't - // registered yet, we should keep track of its PID so we can kill it - // anyway. - // TODO(swang): Evict actor dummy objects as part of actor cleanup. - } - } - - /* Create a new entry and add it to the actor mapping table. TODO(rkn): - * Currently this is never removed (except when the local scheduler state is - * deleted). */ - ActorMapEntry entry; - entry.local_scheduler_id = local_scheduler_id; - entry.driver_id = driver_id; - state->actor_mapping[actor_id] = entry; - - /* Let the scheduling algorithm process the fact that a new actor has been - * created. */ - handle_actor_creation_notification(state, state->algorithm_state, actor_id); -} - -int heartbeat_handler(event_loop *loop, timer_id id, void *context) { - LocalSchedulerState *state = (LocalSchedulerState *) context; - SchedulingAlgorithmState *algorithm_state = state->algorithm_state; - - // Spillback policy invocation is synchronized with the heartbeats. - spillback_tasks_handler(state); - - /* Check that the last heartbeat was not sent too long ago. */ - int64_t current_time = current_time_ms(); - RAY_CHECK(current_time >= state->previous_heartbeat_time); - if (current_time - state->previous_heartbeat_time > - RayConfig::instance().num_heartbeats_timeout() * - RayConfig::instance().heartbeat_timeout_milliseconds()) { - RAY_LOG(FATAL) << "The last heartbeat was sent " - << current_time - state->previous_heartbeat_time - << " milliseconds ago."; - } - state->previous_heartbeat_time = current_time; - - LocalSchedulerInfo info; - /* Ask the scheduling algorithm to fill out the scheduler info struct. */ - provide_scheduler_info(state, algorithm_state, &info); - /* Publish the heartbeat to all subscribers of the local scheduler table. */ - local_scheduler_table_send_info(state->db, &info, NULL); - /* Reset the timer. */ - return RayConfig::instance().heartbeat_timeout_milliseconds(); -} - -void start_server( - const char *node_ip_address, - const char *socket_name, - const char *redis_primary_addr, - int redis_primary_port, - const char *plasma_store_socket_name, - const char *plasma_manager_socket_name, - const char *plasma_manager_address, - bool global_scheduler_exists, - const std::unordered_map &static_resource_conf, - const char *start_worker_command, - int num_workers) { - /* Ignore SIGPIPE signals. If we don't do this, then when we attempt to write - * to a client that has already died, the local scheduler could die. */ - signal(SIGPIPE, SIG_IGN); - /* Ignore SIGCHLD signals. If we don't do this, then worker processes will - * become zombies instead of dying gracefully. */ - signal(SIGCHLD, SIG_IGN); - int fd = bind_ipc_sock(socket_name, true); - event_loop *loop = event_loop_create(); - g_state = LocalSchedulerState_init( - node_ip_address, loop, redis_primary_addr, redis_primary_port, - socket_name, plasma_store_socket_name, plasma_manager_socket_name, - plasma_manager_address, global_scheduler_exists, static_resource_conf, - start_worker_command, num_workers); - /* Register a callback for registering new clients. */ - event_loop_add_file(loop, fd, EVENT_LOOP_READ, new_client_connection, - g_state); - /* Subscribe to receive notifications about tasks that are assigned to this - * local scheduler by the global scheduler or by other local schedulers. - * TODO(rkn): we also need to get any tasks that were assigned to this local - * scheduler before the call to subscribe. */ - if (g_state->db != NULL) { - task_table_subscribe(g_state->db, get_db_client_id(g_state->db), - TaskStatus::SCHEDULED, handle_task_scheduled_callback, - g_state, NULL, NULL, NULL); - } - /* Subscribe to notifications about newly created actors. */ - if (g_state->db != NULL) { - actor_notification_table_subscribe( - g_state->db, handle_actor_creation_callback, g_state, NULL); - } - /* Subscribe to notifications about removed drivers. */ - if (g_state->db != NULL) { - driver_table_subscribe(g_state->db, handle_driver_removed_callback, g_state, - NULL); - } - /* Create a timer for publishing information about the load on the local - * scheduler to the local scheduler table. This message also serves as a - * heartbeat. */ - if (g_state->db != NULL) { - event_loop_add_timer(loop, - RayConfig::instance().heartbeat_timeout_milliseconds(), - heartbeat_handler, g_state); - } - /* Listen for new and deleted db clients. */ - if (g_state->db != NULL) { - db_client_table_cache_init(g_state->db); - } - /* Create a timer for fetching queued tasks' missing object dependencies. */ - event_loop_add_timer( - loop, RayConfig::instance().local_scheduler_fetch_timeout_milliseconds(), - fetch_object_timeout_handler, g_state); - /* Create a timer for initiating the reconstruction of tasks' missing object - * dependencies. */ - event_loop_add_timer( - loop, RayConfig::instance() - .local_scheduler_reconstruction_timeout_milliseconds(), - reconstruct_object_timeout_handler, g_state); - // Create a timer for rerunning actor creation tasks for actor tasks that are - // cached locally. - event_loop_add_timer( - loop, RayConfig::instance() - .local_scheduler_reconstruction_timeout_milliseconds(), - rerun_actor_creation_tasks_timeout_handler, g_state); - /* Run event loop. */ - event_loop_run(loop); -} - -/* Only declare the main function if we are not in testing mode, since the test - * suite has its own declaration of main. */ -#ifndef LOCAL_SCHEDULER_TEST -int main(int argc, char *argv[]) { - InitShutdownRAII ray_log_shutdown_raii( - ray::RayLog::StartRayLog, ray::RayLog::ShutDownRayLog, argv[0], - ray::RayLogLevel::INFO, /*log_dir=*/""); - ray::RayLog::InstallFailureSignalHandler(); - signal(SIGTERM, signal_handler); - /* Path of the listening socket of the local scheduler. */ - char *scheduler_socket_name = NULL; - /* IP address and port of the primary redis instance. */ - char *redis_primary_addr_port = NULL; - /* Socket name for the local Plasma store. */ - char *plasma_store_socket_name = NULL; - /* Socket name for the local Plasma manager. */ - char *plasma_manager_socket_name = NULL; - /* Address for the plasma manager associated with this local scheduler - * instance. */ - char *plasma_manager_address = NULL; - /* The IP address of the node that this local scheduler is running on. */ - char *node_ip_address = NULL; - /* Comma-separated list of configured resource capabilities for this node. */ - char *static_resource_list = NULL; - std::unordered_map static_resource_conf; - /* The command to run when starting new workers. */ - char *start_worker_command = NULL; - /* The number of workers to start. */ - char *num_workers_str = NULL; - int c; - bool global_scheduler_exists = true; - while ((c = getopt(argc, argv, "s:r:p:m:ga:h:c:w:n:")) != -1) { - switch (c) { - case 's': - scheduler_socket_name = optarg; - break; - case 'r': - redis_primary_addr_port = optarg; - break; - case 'p': - plasma_store_socket_name = optarg; - break; - case 'm': - plasma_manager_socket_name = optarg; - break; - case 'g': - global_scheduler_exists = false; - break; - case 'a': - plasma_manager_address = optarg; - break; - case 'h': - node_ip_address = optarg; - break; - case 'c': - static_resource_list = optarg; - break; - case 'w': - start_worker_command = optarg; - break; - case 'n': - num_workers_str = optarg; - break; - default: - RAY_LOG(FATAL) << "unknown option " << c; - } - } - if (!static_resource_list) { - RAY_LOG(FATAL) << "please specify a static resource list with the -c " - << "switch"; - } - // Parse the resource list. - std::istringstream resource_string(static_resource_list); - std::string resource_name; - std::string resource_quantity; - - while (std::getline(resource_string, resource_name, ',')) { - RAY_CHECK(std::getline(resource_string, resource_quantity, ',')); - // TODO(rkn): The line below could throw an exception. What should we do - // about this? - static_resource_conf[resource_name] = std::stod(resource_quantity); - } - - if (!scheduler_socket_name) { - RAY_LOG(FATAL) << "please specify socket for incoming connections with " - << "-s switch"; - } - if (!plasma_store_socket_name) { - RAY_LOG(FATAL) << "please specify socket for connecting to Plasma store " - << "with -p switch"; - } - if (!node_ip_address) { - RAY_LOG(FATAL) << "please specify the node IP address with -h switch"; - } - int num_workers = 0; - if (num_workers_str) { - num_workers = strtol(num_workers_str, NULL, 10); - if (num_workers < 0) { - RAY_LOG(FATAL) << "Number of workers must be nonnegative"; - } - } - - char redis_primary_addr[16]; - char *redis_addr = NULL; - int redis_port = -1; - if (!redis_primary_addr_port) { - /* Start the local scheduler without connecting to Redis. In this case, all - * submitted tasks will be queued and scheduled locally. */ - if (plasma_manager_socket_name) { - RAY_LOG(FATAL) << "if a plasma manager socket name is provided with the " - << "-m switch, then a redis address must be provided with " - << "the -r switch"; - } - } else { - int redis_primary_port; - /* Parse the primary Redis address into an IP address and a port. */ - if (parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr, - &redis_primary_port) == -1) { - RAY_LOG(FATAL) << "if a redis address is provided with the -r switch, it " - << "should be formatted like 127.0.0.1:6379"; - } - if (!plasma_manager_socket_name) { - RAY_LOG(FATAL) << "please specify socket for connecting to Plasma " - << "manager with -m switch"; - } - redis_addr = redis_primary_addr; - redis_port = redis_primary_port; - } - - start_server(node_ip_address, scheduler_socket_name, redis_addr, redis_port, - plasma_store_socket_name, plasma_manager_socket_name, - plasma_manager_address, global_scheduler_exists, - static_resource_conf, start_worker_command, num_workers); -} -#endif diff --git a/src/local_scheduler/local_scheduler.h b/src/local_scheduler/local_scheduler.h deleted file mode 100644 index 39c7523fe..000000000 --- a/src/local_scheduler/local_scheduler.h +++ /dev/null @@ -1,176 +0,0 @@ -#ifndef LOCAL_SCHEDULER_H -#define LOCAL_SCHEDULER_H - -#include "event_loop.h" -#include "local_scheduler_shared.h" -#include "task.h" - -/** - * Establish a connection to a new client. - * - * @param loop Event loop of the local scheduler. - * @param listener_socket Socket the local scheduler is listening on for new - * client requests. - * @param context State of the local scheduler. - * @param events Flag for events that are available on the listener socket. - * @return Void. - */ -void new_client_connection(event_loop *loop, - int listener_sock, - void *context, - int events); - -/** - * Check if a driver is still alive. - * - * @param driver_id The ID of the driver. - * @return True if the driver is still alive and false otherwise. - */ -bool is_driver_alive(WorkerID driver_id); - -/** - * This function can be called by the scheduling algorithm to assign a task - * to a worker. - * - * @param info - * @param task The task that is submitted to the worker. - * @param worker The worker to assign the task to. - * @return Void. - */ -void assign_task_to_worker(LocalSchedulerState *state, - TaskExecutionSpec &task, - LocalSchedulerClient *worker); - -/* - * This function is called whenever a task has finished on one of the workers. - * It updates the resource accounting and the global state store. - * - * @param state The local scheduler state. - * @param worker The worker that finished the task. - * @return Void. - */ -void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker); - -/** - * This is the callback that is used to process a notification from the Plasma - * store that an object has been sealed. - * - * @param loop The local scheduler's event loop. - * @param client_sock The file descriptor to read the notification from. - * @param context The local scheduler state. - * @param events - * @return Void. - */ -void process_plasma_notification(event_loop *loop, - int client_sock, - void *context, - int events); - -/** - * Reconstruct an object. If the object does not exist on any nodes, according - * to the state tables, and if the object is not already being reconstructed, - * this triggers a single reexecution of the task that originally created the - * object. - * - * @param state The local scheduler state. - * @param object_id The ID of the object to reconstruct. - * @return Void. - */ -void reconstruct_object(LocalSchedulerState *state, ObjectID object_id); - -void print_resource_info(const LocalSchedulerState *s, const TaskSpec *spec); - -/** - * Kill a worker, if it is a child process, and clean up all of its associated - * state. Note that this function is also called on drivers, but it should not - * actually send a kill signal to drivers. - * - * @param state The local scheduler state. - * @param worker The local scheduler client to kill. - * @param wait A boolean representing whether to wait for the killed worker to - * exit. - * @param suppress_warning A bool that is true if we should not warn the driver, - * and false otherwise. This should only be true when a driver is - * removed. - * @return Void. - */ -void kill_worker(LocalSchedulerState *state, - LocalSchedulerClient *worker, - bool wait, - bool suppress_warning); - -/** - * Start a worker. This forks a new worker process that can be added to the - * pool of available workers, pending registration of its PID with the local - * scheduler. - * - * @param state The local scheduler state. - * @param Void. - */ -void start_worker(LocalSchedulerState *state); - -/** - * Check if a certain quantity of dynamic resources are available. If num_cpus - * is 0, we ignore the dynamic number of available CPUs (which may be negative). - * - * @param state The state of the local scheduler. - * @param resources The resources to check. - * @return True if there are enough CPUs and GPUs and false otherwise. - */ -bool check_dynamic_resources( - LocalSchedulerState *state, - const std::unordered_map &resources); - -/** - * Acquire additional resources (CPUs and GPUs) for a worker. - * - * @param state The local scheduler state. - * @param worker The worker who is acquiring resources. - * @param resources The resources to acquire. - * @return Void. - */ -void acquire_resources( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - const std::unordered_map &resources); - -/** - * Return resources (CPUs and GPUs) being used by a worker to the local - * scheduler. - * - * @param state The local scheduler state. - * @param worker The worker who is returning resources. - * @param resources The resources to release. - * @return Void. - */ -void release_resources( - LocalSchedulerState *state, - LocalSchedulerClient *worker, - const std::unordered_map &resources); - -/** The following methods are for testing purposes only. */ -#ifdef LOCAL_SCHEDULER_TEST -LocalSchedulerState *LocalSchedulerState_init( - const char *node_ip_address, - event_loop *loop, - const char *redis_addr, - int redis_port, - const char *local_scheduler_socket_name, - const char *plasma_manager_socket_name, - const char *plasma_store_socket_name, - const char *plasma_manager_address, - bool global_scheduler_exists, - const std::unordered_map &static_resource_vector, - const char *worker_path, - int num_workers); - -SchedulingAlgorithmState *get_algorithm_state(LocalSchedulerState *state); - -void process_message(event_loop *loop, - int client_sock, - void *context, - int events); - -#endif - -#endif /* LOCAL_SCHEDULER_H */ diff --git a/src/local_scheduler/local_scheduler_algorithm.cc b/src/local_scheduler/local_scheduler_algorithm.cc deleted file mode 100644 index 89d6c8d6d..000000000 --- a/src/local_scheduler/local_scheduler_algorithm.cc +++ /dev/null @@ -1,1851 +0,0 @@ -#include "local_scheduler_algorithm.h" - -#include -#include -#include - -#include "state/task_table.h" -#include "state/actor_notification_table.h" -#include "state/db_client_table.h" -#include "state/error_table.h" -#include "state/local_scheduler_table.h" -#include "state/object_table.h" -#include "local_scheduler_shared.h" -#include "local_scheduler.h" -#include "common/task.h" - -/* Declared for convenience. */ -void remove_actor(SchedulingAlgorithmState *algorithm_state, ActorID actor_id); - -void give_task_to_global_scheduler(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec); - -void give_task_to_local_scheduler(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - DBClientID local_scheduler_id); - -void clear_missing_dependencies(SchedulingAlgorithmState *algorithm_state, - std::list::iterator it); - -/** A data structure used to track which objects are available locally and - * which objects are being actively fetched. Objects of this type are used for - * both the scheduling algorithm state's local_objects and remote_objects - * tables. An ObjectEntry should be in at most one of the tables and not both - * simultaneously. */ -struct ObjectEntry { - /** A vector of tasks dependent on this object. These tasks are a subset of - * the tasks in the waiting queue. Each element actually stores a reference - * to the corresponding task's queue entry in waiting queue, for fast - * deletion when all of the task's dependencies become available. */ - std::vector::iterator> dependent_tasks; - /** Whether or not to request a transfer of this object. This should be set - * to true for all objects except for actor dummy objects, where the object - * must be generated by executing the task locally. */ - bool request_transfer; -}; - -/** This struct contains information about a specific actor. This struct will be - * used inside of a hash table. */ -typedef struct { - /** The number of tasks that have been executed on this actor so far, per - * handle. This is used to guarantee execution of tasks on actors in the - * order that the tasks were submitted, per handle. Tasks from different - * handles to the same actor may be interleaved. */ - std::unordered_map task_counters; - /** These are the execution dependencies that make up the frontier of the - * actor's runnable tasks. For each actor handle, we store the object ID - * that represents the execution dependency for the next runnable task - * submitted by that handle. */ - std::unordered_map frontier_dependencies; - /** The return value of the most recently executed task. The next task to - * execute should take this as an execution dependency at dispatch time. Set - * to nil if there are no execution dependencies (e.g., this is the first - * task to execute). */ - ObjectID execution_dependency; - /** A queue of tasks to be executed on this actor. The tasks will be sorted by - * the order of their actor counters. */ - std::list *task_queue; - /** The worker that the actor is running on. */ - LocalSchedulerClient *worker; - /** True if the worker is available and false otherwise. */ - bool worker_available; -} LocalActorInfo; - -/** Part of the local scheduler state that is maintained by the scheduling - * algorithm. */ -struct SchedulingAlgorithmState { - /** An array of pointers to tasks that are waiting for dependencies. */ - std::list *waiting_task_queue; - /** An array of pointers to tasks whose dependencies are ready but that are - * waiting to be assigned to a worker. */ - std::list *dispatch_task_queue; - /** This is a hash table from actor ID to information about that actor. In - * particular, a queue of tasks that are waiting to execute on that actor. - * This is only used for actors that exist locally. */ - std::unordered_map local_actor_infos; - /** This is a set of the IDs of the actors that have tasks waiting to run. - * The purpose is to make it easier to dispatch tasks without looping over - * all of the actors. Note that this is an optimization and is not strictly - * necessary. */ - std::unordered_set actors_with_pending_tasks; - /** A vector of actor tasks that have been submitted but this local scheduler - * doesn't know which local scheduler is responsible for them, so cannot - * assign them to the correct local scheduler yet. Whenever a notification - * about a new local scheduler arrives, we will resubmit all of these tasks - * locally. */ - std::vector cached_submitted_actor_tasks; - /** An array of pointers to workers in the worker pool. These are workers - * that have registered a PID with us and that are now waiting to be - * assigned a task to execute. */ - std::vector available_workers; - /** An array of pointers to workers that are currently executing a task, - * unblocked. These are the workers that are leasing some number of - * resources. */ - std::vector executing_workers; - /** An array of pointers to workers that are currently executing a task, - * blocked on some object(s) that isn't available locally yet. These are the - * workers that are executing a task, but that have temporarily returned the - * task's required resources. */ - std::vector blocked_workers; - /** A hash map of the objects that are available in the local Plasma store. - * The key is the object ID. This information could be a little stale. */ - std::unordered_map local_objects; - /** A hash map of the objects that are not available locally. These are - * currently being fetched by this local scheduler. The key is the object - * ID. Every local_scheduler_fetch_timeout_milliseconds, a Plasma fetch - * request will be sent the object IDs in this table. Each entry also holds - * an array of queued tasks that are dependent on it. */ - std::unordered_map remote_objects; -}; - -SchedulingAlgorithmState *SchedulingAlgorithmState_init(void) { - SchedulingAlgorithmState *algorithm_state = new SchedulingAlgorithmState(); - /* Initialize the local data structures used for queuing tasks and workers. */ - algorithm_state->waiting_task_queue = new std::list(); - algorithm_state->dispatch_task_queue = new std::list(); - - return algorithm_state; -} - -void SchedulingAlgorithmState_free(SchedulingAlgorithmState *algorithm_state) { - /* Free all of the tasks in the waiting queue. */ - delete algorithm_state->waiting_task_queue; - /* Free all the tasks in the dispatch queue. */ - delete algorithm_state->dispatch_task_queue; - /* Remove all of the remaining actors. */ - while (algorithm_state->local_actor_infos.size() != 0) { - auto it = algorithm_state->local_actor_infos.begin(); - ActorID actor_id = it->first; - remove_actor(algorithm_state, actor_id); - } - /* Free the algorithm state. */ - delete algorithm_state; -} - -/** - * This is a helper method to check if a worker is in a vector of workers. - * - * @param worker_vector A vector of workers. - * @param The worker to look for in the vector. - * @return True if the worker is in the vector and false otherwise. - */ -bool worker_in_vector(std::vector &worker_vector, - LocalSchedulerClient *worker) { - auto it = std::find(worker_vector.begin(), worker_vector.end(), worker); - return it != worker_vector.end(); -} - -/** - * This is a helper method to remove a worker from a vector of workers if it is - * present in the vector. - * - * @param worker_vector A vector of workers. - * @param The worker to remove. - * @return True if the worker was removed and false otherwise. - */ -bool remove_worker_from_vector( - std::vector &worker_vector, - LocalSchedulerClient *worker) { - /* Find the worker in the list of executing workers. */ - auto it = std::find(worker_vector.begin(), worker_vector.end(), worker); - bool remove_worker = (it != worker_vector.end()); - if (remove_worker) { - /* Remove the worker from the list of workers. */ - using std::swap; - swap(*it, worker_vector.back()); - worker_vector.pop_back(); - } - return remove_worker; -} - -void provide_scheduler_info(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerInfo *info) { - info->total_num_workers = state->workers.size(); - /* TODO(swang): Provide separate counts for tasks that are waiting for - * dependencies vs tasks that are waiting to be assigned. */ - int64_t waiting_task_queue_length = - algorithm_state->waiting_task_queue->size(); - int64_t dispatch_task_queue_length = - algorithm_state->dispatch_task_queue->size(); - info->task_queue_length = - waiting_task_queue_length + dispatch_task_queue_length; - info->available_workers = algorithm_state->available_workers.size(); - /* Copy static and dynamic resource information. */ - info->dynamic_resources = state->dynamic_resources; - info->static_resources = state->static_resources; -} - -/** - * Create the LocalActorInfo struct for an actor worker that this local - * scheduler is responsible for. For a given actor, this will either be done - * when the first task for that actor arrives or when the worker running that - * actor connects to the local scheduler. - * - * @param algorithm_state The state of the scheduling algorithm. - * @param actor_id The actor ID of the actor being created. - * @param initial_execution_dependency The dummy object ID of the actor - * creation task. - * @param worker The worker struct for the worker that is running this actor. - * If the worker struct has not been created yet (meaning that the worker - * that is running this actor has not registered with the local scheduler - * yet, and so create_actor is being called because a task for that actor - * has arrived), then this should be NULL. - * @return Void. - */ -void create_actor(SchedulingAlgorithmState *algorithm_state, - const ActorID &actor_id, - const ObjectID &initial_execution_dependency, - LocalSchedulerClient *worker) { - LocalActorInfo entry; - entry.task_counters[ActorHandleID::nil()] = 0; - entry.frontier_dependencies[ActorHandleID::nil()] = ObjectID::nil(); - /* The actor has not yet executed any tasks, so there are no execution - * dependencies for the next task to be scheduled. */ - entry.execution_dependency = initial_execution_dependency; - entry.task_queue = new std::list(); - entry.worker = worker; - entry.worker_available = false; - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) == 0); - algorithm_state->local_actor_infos[actor_id] = entry; - - /* Log some useful information about the actor that we created. */ - RAY_LOG(DEBUG) << "Creating actor with ID " << actor_id; -} - -void remove_actor(SchedulingAlgorithmState *algorithm_state, ActorID actor_id) { - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) == 1); - LocalActorInfo &entry = - algorithm_state->local_actor_infos.find(actor_id)->second; - - /* Log some useful information about the actor that we're removing. */ - size_t count = entry.task_queue->size(); - if (count > 0) { - RAY_LOG(WARNING) << "Removing actor with ID " << actor_id << " and " - << count << " remaining tasks."; - } - - entry.task_queue->clear(); - delete entry.task_queue; - /* Remove the entry from the hash table. */ - algorithm_state->local_actor_infos.erase(actor_id); - - /* Remove the actor ID from the set of actors with pending tasks. */ - algorithm_state->actors_with_pending_tasks.erase(actor_id); -} - -/** - * Dispatch a task to an actor if possible. - * - * @param state The state of the local scheduler. - * @param algorithm_state The state of the scheduling algorithm. - * @param actor_id The ID of the actor corresponding to the worker. - * @return True if a task was dispatched to the actor and false otherwise. - */ -bool dispatch_actor_task(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id) { - /* Make sure this worker actually is an actor. */ - RAY_CHECK(!actor_id.is_nil()); - /* Return if this actor doesn't have any pending tasks. */ - if (algorithm_state->actors_with_pending_tasks.find(actor_id) == - algorithm_state->actors_with_pending_tasks.end()) { - return false; - } - /* Make sure this actor belongs to this local scheduler. */ - if (state->actor_mapping.count(actor_id) != 1) { - /* The creation notification for this actor has not yet arrived at the local - * scheduler. This should be rare. */ - return false; - } - RAY_CHECK(state->actor_mapping[actor_id].local_scheduler_id == - get_db_client_id(state->db)); - - /* Get the local actor entry for this actor. */ - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) != 0); - LocalActorInfo &entry = - algorithm_state->local_actor_infos.find(actor_id)->second; - - /* There should be some queued tasks for this actor. */ - RAY_CHECK(!entry.task_queue->empty()); - /* If the worker is not available, we cannot assign a task to it. */ - if (!entry.worker_available) { - return false; - } - - /* Check whether we can execute the first task in the queue. */ - auto task = entry.task_queue->begin(); - TaskSpec *spec = task->Spec(); - ActorHandleID next_task_handle_id = TaskSpec_actor_handle_id(spec); - /* We can only execute tasks in order of task_counter. */ - if (TaskSpec_actor_counter(spec) != - entry.task_counters[next_task_handle_id]) { - return false; - } - - /* If there are not enough resources available, we cannot assign the task. */ - RAY_CHECK(0 == TaskSpec_get_required_resource(spec, "GPU")); - if (!check_dynamic_resources(state, TaskSpec_get_required_resources(spec))) { - return false; - } - - /* Update the task's execution dependencies to reflect the actual execution - * order to support deterministic reconstruction. */ - /* NOTE(swang): The update of an actor task's execution dependencies is - * performed asynchronously. This means that if this local scheduler dies, we - * may lose updates that are in flight to the task table. We only guarantee - * deterministic reconstruction ordering for tasks whose updates are - * reflected in the task table. */ - std::vector ordered_execution_dependencies; - ordered_execution_dependencies.push_back(entry.execution_dependency); - task->SetExecutionDependencies(ordered_execution_dependencies); - - /* Assign the first task in the task queue to the worker and mark the worker - * as unavailable. */ - assign_task_to_worker(state, *task, entry.worker); - entry.execution_dependency = TaskSpec_actor_dummy_object(spec); - entry.worker_available = false; - /* Extend the frontier to include the assigned task. */ - entry.task_counters[next_task_handle_id] += 1; - entry.frontier_dependencies[next_task_handle_id] = entry.execution_dependency; - - /* Remove the task from the actor's task queue. */ - entry.task_queue->erase(task); - /* If there are no more tasks in the queue, then indicate that the actor has - * no tasks. */ - if (entry.task_queue->empty()) { - algorithm_state->actors_with_pending_tasks.erase(actor_id); - } - - return true; -} - -void handle_convert_worker_to_actor( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - const ActorID &actor_id, - const ObjectID &initial_execution_dependency, - LocalSchedulerClient *worker) { - if (algorithm_state->local_actor_infos.count(actor_id) == 0) { - create_actor(algorithm_state, actor_id, initial_execution_dependency, - worker); - } else { - /* In this case, the LocalActorInfo struct was already been created by the - * first call to add_task_to_actor_queue. However, the worker field was not - * filled out, so fill out the correct worker field now. */ - algorithm_state->local_actor_infos[actor_id].worker = worker; - } - /* Increment the task counter for the creator's handle to account for the - * actor creation task. */ - auto &task_counters = - algorithm_state->local_actor_infos[actor_id].task_counters; - RAY_CHECK(task_counters[ActorHandleID::nil()] == 0); - task_counters[ActorHandleID::nil()]++; -} - -/** - * Finishes a killed task by inserting dummy objects for each of its returns. - */ -void finish_killed_task(LocalSchedulerState *state, - TaskExecutionSpec &execution_spec) { - TaskSpec *spec = execution_spec.Spec(); - int64_t num_returns = TaskSpec_num_returns(spec); - for (int i = 0; i < num_returns; i++) { - ObjectID object_id = TaskSpec_return(spec, i); - std::shared_ptr data; - // TODO(ekl): this writes an invalid arrow object, which is sufficient to - // signal that the worker failed, but it would be nice to return more - // detailed failure metadata in the future. - arrow::Status status = - state->plasma_conn->Create(object_id.to_plasma_id(), 1, NULL, 0, &data); - if (!status.IsPlasmaObjectExists()) { - ARROW_CHECK_OK(status); - ARROW_CHECK_OK(state->plasma_conn->Seal(object_id.to_plasma_id())); - } - } - /* Mark the task as done. */ - if (state->db != NULL) { - Task *task = Task_alloc(execution_spec, TaskStatus::DONE, - get_db_client_id(state->db)); - // In most cases, task_table_update would be appropriate, however, it is - // possible in some cases that the task has not yet been added to the task - // table (e.g., if it is an actor task that is queued locally because the - // actor has not been created yet). - task_table_add_task(state->db, task, NULL, NULL, NULL); - } -} - -/** - * Insert a task queue entry into an actor's dispatch queue. The task is - * inserted in sorted order by task counter. If this is the first task - * scheduled to this actor and the worker process has not yet connected, then - * this also creates a LocalActorInfo entry for the actor. - * - * @param state The state of the local scheduler. - * @param algorithm_state The state of the scheduling algorithm. - * @param task_entry The task queue entry to add to the actor's queue. - * @return Void. - */ -void insert_actor_task_queue(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec task_entry) { - TaskSpec *spec = task_entry.Spec(); - /* Get the local actor entry for this actor. */ - ActorID actor_id = TaskSpec_actor_id(spec); - ActorHandleID task_handle_id = TaskSpec_actor_handle_id(spec); - int64_t task_counter = TaskSpec_actor_counter(spec); - - /* Fail the task immediately; it's destined for a dead actor. */ - if (state->removed_actors.find(actor_id) != state->removed_actors.end()) { - finish_killed_task(state, task_entry); - return; - } - - LocalActorInfo &entry = - algorithm_state->local_actor_infos.find(actor_id)->second; - if (entry.task_counters.count(task_handle_id) == 0) { - entry.task_counters[task_handle_id] = 0; - } - /* Extend the frontier to include the new handle. */ - if (entry.frontier_dependencies.count(task_handle_id) == 0) { - RAY_CHECK(task_entry.ExecutionDependencies().size() == 1); - entry.frontier_dependencies[task_handle_id] = - task_entry.ExecutionDependencies()[0]; - } - - /* As a sanity check, the counter of the new task should be greater than the - * number of tasks that have executed on this actor so far (since we are - * guaranteeing in-order execution of the tasks on the actor). TODO(rkn): This - * check will fail if the fault-tolerance mechanism resubmits a task on an - * actor. */ - if (task_counter < entry.task_counters[task_handle_id]) { - RAY_LOG(INFO) << "A task that has already been executed has been " - << "resubmitted, so we are ignoring it. This should only " - << "happen during reconstruction."; - return; - } - - /* Insert the task spec to the actor's task queue in sorted order, per actor - * handle ID. Find the first task in the queue with a counter greater than - * the submitted task's and the same handle ID. */ - auto it = entry.task_queue->begin(); - for (; it != entry.task_queue->end(); it++) { - TaskSpec *pending_task_spec = it->Spec(); - /* Skip tasks submitted by a different handle. */ - if (!(task_handle_id == TaskSpec_actor_handle_id(pending_task_spec))) { - continue; - } - /* A duplicate task submitted by the same handle. */ - if (task_counter == TaskSpec_actor_counter(pending_task_spec)) { - RAY_LOG(INFO) << "A task was resubmitted, so we are ignoring it. This " - << "should only happen during reconstruction."; - return; - } - /* We found a task with the same handle ID and a greater task counter. */ - if (task_counter < TaskSpec_actor_counter(pending_task_spec)) { - break; - } - } - entry.task_queue->insert(it, std::move(task_entry)); - - /* Record the fact that this actor has a task waiting to execute. */ - algorithm_state->actors_with_pending_tasks.insert(actor_id); -} - -/** - * Queue a task to be dispatched for an actor. Update the task table for the - * queued task. TODO(rkn): Should we also update the task table in the case - * where the tasks are cached locally? - * - * @param state The state of the local scheduler. - * @param algorithm_state The state of the scheduling algorithm. - * @param spec The task spec to add. - * @param from_global_scheduler True if the task was assigned to this local - * scheduler by the global scheduler and false if it was submitted - * locally by a worker. - * @return Void. - */ -void queue_actor_task(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - bool from_global_scheduler) { - TaskSpec *spec = execution_spec.Spec(); - ActorID actor_id = TaskSpec_actor_id(spec); - RAY_CHECK(!actor_id.is_nil()); - - /* Update the task table. */ - if (state->db != NULL) { - Task *task = Task_alloc(execution_spec, TaskStatus::QUEUED, - get_db_client_id(state->db)); - if (from_global_scheduler) { - /* If the task is from the global scheduler, it's already been added to - * the task table, so just update the entry. */ - task_table_update(state->db, task, NULL, NULL, NULL); - } else { - /* Otherwise, this is the first time the task has been seen in the - * system (unless it's a resubmission of a previous task), so add the - * entry. */ - task_table_add_task(state->db, task, NULL, NULL, NULL); - } - } - - // Create a new task queue entry. This must come after the above block because - // insert_actor_task_queue may call task_table_update internally, which must - // come after the prior call to task_table_add_task. - TaskExecutionSpec copy = TaskExecutionSpec(&execution_spec); - insert_actor_task_queue(state, algorithm_state, std::move(copy)); -} - -/** - * Fetch a queued task's missing object dependency. The fetch request will be - * retried every local_scheduler_fetch_timeout_milliseconds until the object is - * available locally. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param task_entry_it A reference to the task entry in the waiting queue. - * @param obj_id The ID of the object that the task is dependent on. - * @param request_transfer Whether to request a transfer of this object from - * other plasma managers. This should be set to false for execution - * dependencies, which should be fulfilled by executing the - * corresponding task locally. - * @returns Void. - */ -void fetch_missing_dependency( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - std::list::iterator task_entry_it, - plasma::ObjectID obj_id, - bool request_transfer) { - if (algorithm_state->remote_objects.count(obj_id) == 0) { - /* We weren't actively fetching this object. Try the fetch once - * immediately. */ - if (state->plasma_conn->get_manager_fd() != -1) { - auto arrow_status = state->plasma_conn->Fetch(1, &obj_id); - if (!arrow_status.ok()) { - LocalSchedulerState_free(state); - /* TODO(swang): Local scheduler should also exit even if there are no - * pending fetches. This could be done by subscribing to the db_client - * table, or pinging the plasma manager in the heartbeat handler. */ - RAY_LOG(FATAL) << "Lost connection to the plasma manager, local " - << "scheduler is exiting. Error: " - << arrow_status.ToString(); - } - } - /* Create an entry and add it to the list of active fetch requests to - * ensure that the fetch actually happens. The entry will be moved to the - * hash table of locally available objects in handle_object_available when - * the object becomes available locally. It will get freed if the object is - * subsequently removed locally. */ - ObjectEntry entry; - entry.request_transfer = request_transfer; - algorithm_state->remote_objects[obj_id] = entry; - } - algorithm_state->remote_objects[obj_id].dependent_tasks.push_back( - task_entry_it); -} - -/** - * Fetch a queued task's missing object dependencies. The fetch requests will - * be retried every local_scheduler_fetch_timeout_milliseconds until all - * objects are available locally. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param task_entry_it A reference to the task entry in the waiting queue. - * @returns Void. - */ -void fetch_missing_dependencies( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - std::list::iterator task_entry_it) { - int64_t num_dependencies = task_entry_it->NumDependencies(); - int num_missing_dependencies = 0; - for (int64_t i = 0; i < num_dependencies; ++i) { - int count = task_entry_it->DependencyIdCount(i); - for (int j = 0; j < count; ++j) { - ObjectID obj_id = task_entry_it->DependencyId(i, j); - /* If the entry is not yet available locally, record the dependency. */ - if (algorithm_state->local_objects.count(obj_id) == 0) { - /* Do not request a transfer from other plasma managers if this is an - * execution dependency. */ - bool request_transfer = task_entry_it->IsStaticDependency(i); - fetch_missing_dependency(state, algorithm_state, task_entry_it, - obj_id.to_plasma_id(), request_transfer); - ++num_missing_dependencies; - } - } - } - RAY_CHECK(num_missing_dependencies > 0); -} - -/** - * Clear a queued task's missing object dependencies. This is the inverse of - * fetch_missing_dependencies. - * TODO(swang): Test this function. - * - * @param algorithm_state The scheduling algorithm state. - * @param task_entry_it A reference to the task entry in the waiting queue. - * @returns Void. - */ -void clear_missing_dependencies( - SchedulingAlgorithmState *algorithm_state, - std::list::iterator task_entry_it) { - int64_t num_dependencies = task_entry_it->NumDependencies(); - for (int64_t i = 0; i < num_dependencies; ++i) { - int count = task_entry_it->DependencyIdCount(i); - for (int j = 0; j < count; ++j) { - ObjectID obj_id = task_entry_it->DependencyId(i, j); - /* If this object dependency is missing, remove this task from the - * object's list of dependent tasks. */ - auto entry = algorithm_state->remote_objects.find(obj_id); - if (entry != algorithm_state->remote_objects.end()) { - /* Find and remove the given task. */ - auto &dependent_tasks = entry->second.dependent_tasks; - for (auto dependent_task_it = dependent_tasks.begin(); - dependent_task_it != dependent_tasks.end();) { - if (*dependent_task_it == task_entry_it) { - dependent_task_it = dependent_tasks.erase(dependent_task_it); - } else { - dependent_task_it++; - } - } - /* If the missing object dependency has no more dependent tasks, then - * remove it. */ - if (dependent_tasks.empty()) { - algorithm_state->remote_objects.erase(entry); - } - } - } - } -} - -/** - * Check if all of the remote object arguments for a task are available in the - * local object store. - * - * @param algorithm_state The scheduling algorithm state. - * @param task Task specification of the task to check. - * @return bool This returns true if all of the remote object arguments for the - * task are present in the local object store, otherwise it returns - * false. - */ -bool can_run(SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &task) { - int64_t num_dependencies = task.NumDependencies(); - for (int i = 0; i < num_dependencies; ++i) { - int count = task.DependencyIdCount(i); - for (int j = 0; j < count; ++j) { - ObjectID obj_id = task.DependencyId(i, j); - if (algorithm_state->local_objects.count(obj_id) == 0) { - /* The object is not present locally, so this task cannot be scheduled - * right now. */ - return false; - } - } - } - return true; -} - -bool object_locally_available(SchedulingAlgorithmState *algorithm_state, - ObjectID object_id) { - return algorithm_state->local_objects.count(object_id) == 1; -} - -/* TODO(swang): This method is not covered by any valgrind tests. */ -int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) { - int64_t start_time = current_time_ms(); - - LocalSchedulerState *state = (LocalSchedulerState *) context; - /* Only try the fetches if we are connected to the object store manager. */ - if (state->plasma_conn->get_manager_fd() == -1) { - RAY_LOG(INFO) - << "Local scheduler is not connected to a object store manager"; - return RayConfig::instance().local_scheduler_fetch_timeout_milliseconds(); - } - - std::vector object_id_vec; - for (auto const &entry : state->algorithm_state->remote_objects) { - if (entry.second.request_transfer) { - object_id_vec.push_back(entry.first); - } - } - - ObjectID *object_ids = object_id_vec.data(); - int64_t num_object_ids = object_id_vec.size(); - - /* Divide very large fetch requests into smaller fetch requests so that a - * single fetch request doesn't block the plasma manager for a long time. */ - for (int64_t j = 0; j < num_object_ids; - j += RayConfig::instance().local_scheduler_fetch_request_size()) { - int num_objects_in_request = - std::min( - num_object_ids, - j + RayConfig::instance().local_scheduler_fetch_request_size()) - - j; - auto arrow_status = state->plasma_conn->Fetch( - num_objects_in_request, - reinterpret_cast(&object_ids[j])); - if (!arrow_status.ok()) { - LocalSchedulerState_free(state); - RAY_LOG(FATAL) << "Lost connection to the plasma manager, local " - << "scheduler is exiting. Error: " - << arrow_status.ToString(); - } - } - - /* Print a warning if this method took too long. */ - int64_t end_time = current_time_ms(); - if (end_time - start_time > - RayConfig::instance().max_time_for_handler_milliseconds()) { - RAY_LOG(WARNING) << "fetch_object_timeout_handler took " - << end_time - start_time << " milliseconds."; - } - - /* Wait at least local_scheduler_fetch_timeout_milliseconds before running - * this timeout handler again. But if we're waiting for a large number of - * objects, wait longer (e.g., 10 seconds for one million objects) so that we - * don't overwhelm the plasma manager. */ - return std::max( - RayConfig::instance().local_scheduler_fetch_timeout_milliseconds(), - int64_t(0.01 * num_object_ids)); -} - -/* TODO(swang): This method is not covered by any valgrind tests. */ -int reconstruct_object_timeout_handler(event_loop *loop, - timer_id id, - void *context) { - int64_t start_time = current_time_ms(); - - LocalSchedulerState *state = (LocalSchedulerState *) context; - - /* This vector is used to track which object IDs to reconstruct next. If the - * vector is empty, we repopulate it with all of the keys of the remote object - * table. During every pass through this handler, we call reconstruct on up to - * max_num_to_reconstruct elements of the vector (after first checking that - * the object IDs are still missing). */ - static std::vector object_ids_to_reconstruct; - - /* If the set is empty, repopulate it. */ - if (object_ids_to_reconstruct.size() == 0) { - for (auto const &entry : state->algorithm_state->remote_objects) { - object_ids_to_reconstruct.push_back(entry.first); - } - } - - int64_t num_reconstructed = 0; - for (size_t i = 0; i < object_ids_to_reconstruct.size(); i++) { - ObjectID object_id = object_ids_to_reconstruct[i]; - /* Only call reconstruct if we are still missing the object. */ - if (state->algorithm_state->remote_objects.find(object_id) != - state->algorithm_state->remote_objects.end()) { - reconstruct_object(state, object_id); - } - num_reconstructed++; - if (num_reconstructed == RayConfig::instance().max_num_to_reconstruct()) { - break; - } - } - object_ids_to_reconstruct.erase( - object_ids_to_reconstruct.begin(), - object_ids_to_reconstruct.begin() + num_reconstructed); - - /* Print a warning if this method took too long. */ - int64_t end_time = current_time_ms(); - if (end_time - start_time > - RayConfig::instance().max_time_for_handler_milliseconds()) { - RAY_LOG(WARNING) << "reconstruct_object_timeout_handler took " - << end_time - start_time << " milliseconds."; - } - - return RayConfig::instance() - .local_scheduler_reconstruction_timeout_milliseconds(); -} - -int rerun_actor_creation_tasks_timeout_handler(event_loop *loop, - timer_id id, - void *context) { - int64_t start_time = current_time_ms(); - - LocalSchedulerState *state = (LocalSchedulerState *) context; - - // Create a set of the dummy object IDs for the actor creation tasks to - // reconstruct. - std::unordered_set actor_dummy_objects; - for (auto const &execution_spec : - state->algorithm_state->cached_submitted_actor_tasks) { - ObjectID actor_creation_dummy_object_id = - TaskSpec_actor_creation_dummy_object_id(execution_spec.Spec()); - actor_dummy_objects.insert(actor_creation_dummy_object_id); - } - - // Issue reconstruct calls. - for (auto const &object_id : actor_dummy_objects) { - reconstruct_object(state, object_id); - } - - // Print a warning if this method took too long. - int64_t end_time = current_time_ms(); - if (end_time - start_time > - RayConfig::instance().max_time_for_handler_milliseconds()) { - RAY_LOG(WARNING) << "reconstruct_object_timeout_handler took " - << end_time - start_time << " milliseconds."; - } - - return RayConfig::instance() - .local_scheduler_reconstruction_timeout_milliseconds(); -} - -/** - * Return true if there are still some resources available and false otherwise. - * - * @param state The scheduler state. - * @return True if there are still some resources and false if there are not. - */ -bool resources_available(LocalSchedulerState *state) { - bool resources_available = false; - for (auto const &resource_pair : state->dynamic_resources) { - if (resource_pair.second > 0) { - resources_available = true; - } - } - return resources_available; -} - -void spillback_tasks_handler(LocalSchedulerState *state) { - SchedulingAlgorithmState *algorithm_state = state->algorithm_state; - - int64_t num_to_spillback = std::min( - static_cast(algorithm_state->dispatch_task_queue->size()), - RayConfig::instance().max_tasks_to_spillback()); - - auto it = algorithm_state->dispatch_task_queue->end(); - for (int64_t i = 0; i < num_to_spillback; i++) { - it--; - } - - for (int64_t i = 0; i < num_to_spillback; i++) { - it->IncrementSpillbackCount(); - // If an actor hasn't been created for a while, push a warning to the - // driver. - if (it->SpillbackCount() % - RayConfig::instance().actor_creation_num_spillbacks_warning() == - 0) { - TaskSpec *spec = it->Spec(); - if (TaskSpec_is_actor_creation_task(spec)) { - std::ostringstream error_message; - error_message << "The actor with ID " - << TaskSpec_actor_creation_id(spec) << " is taking a " - << "while to be created. It is possible that the " - << "cluster does not have enough resources to place this " - << "actor (this may be normal while an autoscaling " - << "is scaling up). Consider reducing the number of " - << "actors created, or " - << "increasing the number of slots available by using " - << "the --num-cpus, --num-gpus, and --resources flags. " - << "The actor creation task is requesting "; - for (auto const &resource_pair : - TaskSpec_get_required_resources(spec)) { - error_message << resource_pair.second << " " << resource_pair.first - << " "; - } - push_error(state->db, TaskSpec_driver_id(spec), - ErrorIndex::ACTOR_NOT_CREATED, error_message.str()); - } - } - - give_task_to_global_scheduler(state, algorithm_state, *it); - // Dequeue the task. - it = algorithm_state->dispatch_task_queue->erase(it); - } -} - -/** - * Assign as many tasks from the dispatch queue as possible. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @return Void. - */ -void dispatch_tasks(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state) { - /* Assign as many tasks as we can, while there are workers available. */ - for (auto it = algorithm_state->dispatch_task_queue->begin(); - it != algorithm_state->dispatch_task_queue->end();) { - TaskSpec *spec = it->Spec(); - /* If there is a task to assign, but there are no more available workers in - * the worker pool, then exit. Ensure that there will be an available - * worker during a future invocation of dispatch_tasks. */ - if (algorithm_state->available_workers.size() == 0) { - if (state->child_pids.size() == 0) { - /* If there are no workers, including those pending PID registration, - * then we must start a new one to replenish the worker pool. */ - start_worker(state); - } - return; - } - - /* Terminate early if there are no more resources available. */ - if (!resources_available(state)) { - return; - } - - /* Skip to the next task if this task cannot currently be satisfied. */ - if (!check_dynamic_resources(state, - TaskSpec_get_required_resources(spec))) { - /* This task could not be satisfied -- proceed to the next task. */ - ++it; - continue; - } - - /* Dispatch this task to an available worker and dequeue the task. */ - RAY_LOG(DEBUG) << "Dispatching task"; - /* Get the last available worker in the available worker queue. */ - LocalSchedulerClient *worker = algorithm_state->available_workers.back(); - /* Tell the available worker to execute the task. */ - assign_task_to_worker(state, *it, worker); - /* Remove the worker from the available queue, and add it to the executing - * workers. */ - algorithm_state->available_workers.pop_back(); - algorithm_state->executing_workers.push_back(worker); - print_resource_info(state, spec); - /* Dequeue the task. */ - it = algorithm_state->dispatch_task_queue->erase(it); - } /* End for each task in the dispatch queue. */ -} - -/** - * Attempt to dispatch both regular tasks and actor tasks. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @return Void. - */ -void dispatch_all_tasks(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state) { - /* First attempt to dispatch regular tasks. */ - dispatch_tasks(state, algorithm_state); - - /* Attempt to dispatch actor tasks. */ - auto it = algorithm_state->actors_with_pending_tasks.begin(); - while (it != algorithm_state->actors_with_pending_tasks.end()) { - // We cannot short-circuit and exit here if there are no resources - // available because actor methods may require 0 CPUs. - - /* We increment the iterator ahead of time because the call to - * dispatch_actor_task may invalidate the current iterator. */ - ActorID actor_id = *it; - it++; - /* Dispatch tasks for the current actor. */ - dispatch_actor_task(state, algorithm_state, actor_id); - } -} - -/** - * A helper function to allocate a queue entry for a task specification and - * push it onto a generic queue. - * - * @param state The state of the local scheduler. - * @param task_queue A pointer to a task queue. NOTE: Because we are using - * utlist.h, we must pass in a pointer to the queue we want to append - * to. If we passed in the queue itself and the queue was empty, this - * would append the task to a queue that we don't have a reference to. - * @param task_entry A pointer to the task entry to queue. - * @param from_global_scheduler Whether or not the task was from a global - * scheduler. If false, the task was submitted by a worker. - * @return A reference to the entry in the queue that was pushed. - */ -std::list::iterator queue_task( - LocalSchedulerState *state, - std::list *task_queue, - TaskExecutionSpec &task_entry, - bool from_global_scheduler) { - /* The task has been added to a local scheduler queue. Write the entry in the - * task table to notify others that we have queued it. */ - if (state->db != NULL) { - Task *task = - Task_alloc(task_entry, TaskStatus::QUEUED, get_db_client_id(state->db)); - if (from_global_scheduler) { - /* If the task is from the global scheduler, it's already been added to - * the task table, so just update the entry. */ - task_table_update(state->db, task, NULL, NULL, NULL); - } else { - /* Otherwise, this is the first time the task has been seen in the system - * (unless it's a resubmission of a previous task), so add the entry. */ - task_table_add_task(state->db, task, NULL, NULL, NULL); - } - } - - /* Copy the spec and add it to the task queue. The allocated spec will be - * freed when it is assigned to a worker. */ - TaskExecutionSpec copy = TaskExecutionSpec(&task_entry); - task_queue->push_back(std::move(copy)); - /* Since we just queued the task, we can get a reference to it by going to - * the last element in the queue. */ - auto it = task_queue->end(); - --it; - - return it; -} - -/** - * Queue a task whose dependencies are missing. When the task's object - * dependencies become available, the task will be moved to the dispatch queue. - * If we have a connection to a plasma manager, begin trying to fetch the - * dependencies. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param spec The task specification to queue. - * @param from_global_scheduler Whether or not the task was from a global - * scheduler. If false, the task was submitted by a worker. - * @return Void. - */ -void queue_waiting_task(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - bool from_global_scheduler) { - /* For actor tasks, do not queue tasks that have already been executed. */ - auto spec = execution_spec.Spec(); - if (!TaskSpec_actor_id(spec).is_nil()) { - auto entry = - algorithm_state->local_actor_infos.find(TaskSpec_actor_id(spec)); - if (entry != algorithm_state->local_actor_infos.end()) { - /* Find the highest task counter with the same handle ID as the task to - * queue. */ - auto &task_counters = entry->second.task_counters; - auto task_counter = task_counters.find(TaskSpec_actor_handle_id(spec)); - if (task_counter != task_counters.end() && - TaskSpec_actor_counter(spec) < task_counter->second) { - /* If the task to queue has a lower task counter, do not queue it. */ - RAY_LOG(INFO) << "A task that has already been executed has been " - << "resubmitted, so we are ignoring it. This should only " - << "happen during reconstruction."; - return; - } - } - } - - RAY_LOG(DEBUG) << "Queueing task in waiting queue"; - auto it = queue_task(state, algorithm_state->waiting_task_queue, - execution_spec, from_global_scheduler); - fetch_missing_dependencies(state, algorithm_state, it); -} - -/** - * Queue a task whose dependencies are ready. When the task reaches the front - * of the dispatch queue and workers are available, it will be assigned. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param spec The task specification to queue. - * @param from_global_scheduler Whether or not the task was from a global - * scheduler. If false, the task was submitted by a worker. - * @return Void. - */ -void queue_dispatch_task(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - bool from_global_scheduler) { - RAY_LOG(DEBUG) << "Queueing task in dispatch queue"; - TaskSpec *spec = execution_spec.Spec(); - if (TaskSpec_is_actor_task(spec)) { - queue_actor_task(state, algorithm_state, execution_spec, - from_global_scheduler); - } else { - queue_task(state, algorithm_state->dispatch_task_queue, execution_spec, - from_global_scheduler); - } -} - -/** - * Add the task to the proper local scheduler queue. This assumes that the - * scheduling decision to place the task on this node has already been made, - * whether locally or by the global scheduler. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param spec The task specification to queue. - * @param from_global_scheduler Whether or not the task was from a global - * scheduler. If false, the task was submitted by a worker. - * @return Void. - */ -void queue_task_locally(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - bool from_global_scheduler) { - if (can_run(algorithm_state, execution_spec)) { - /* Dependencies are ready, so push the task to the dispatch queue. */ - queue_dispatch_task(state, algorithm_state, execution_spec, - from_global_scheduler); - } else { - /* Dependencies are not ready, so push the task to the waiting queue. */ - queue_waiting_task(state, algorithm_state, execution_spec, - from_global_scheduler); - } -} - -void give_task_to_local_scheduler_retry(UniqueID id, - void *user_context, - void *user_data) { - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - Task *task = (Task *) user_data; - RAY_CHECK(Task_state(task) == TaskStatus::SCHEDULED); - - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - TaskSpec *spec = execution_spec->Spec(); - RAY_CHECK(TaskSpec_is_actor_task(spec)); - - ActorID actor_id = TaskSpec_actor_id(spec); - - if (state->actor_mapping.count(actor_id) == 0) { - // Process the actor task submission again. This will cache the task - // locally until a new actor creation notification is broadcast. We will - // attempt to reissue the actor creation tasks for all cached actor tasks - // in rerun_actor_creation_tasks_timeout_handler. - handle_actor_task_submitted(state, state->algorithm_state, *execution_spec); - return; - } - - DBClientID remote_local_scheduler_id = - state->actor_mapping[actor_id].local_scheduler_id; - - // TODO(rkn): db_client_table_cache_get is a blocking call, is this a - // performance issue? - DBClient remote_local_scheduler = - db_client_table_cache_get(state->db, remote_local_scheduler_id); - - // Check if the local scheduler that we're assigning this task to is still - // alive. - if (remote_local_scheduler.is_alive) { - // The local scheduler is still alive, which means that perhaps it hasn't - // subscribed to the appropriate channel yet, so retrying should suffice. - // This should be rare. - give_task_to_local_scheduler( - state, state->algorithm_state, *execution_spec, - state->actor_mapping[actor_id].local_scheduler_id); - } else { - // The local scheduler is dead, so we will need to recreate the actor by - // invoking reconstruction. - RAY_LOG(INFO) << "Local scheduler " << remote_local_scheduler_id - << " that was running actor " << actor_id << " died."; - RAY_CHECK(state->actor_mapping.count(actor_id) == 1); - // Update the actor mapping. - state->actor_mapping.erase(actor_id); - // Process the actor task submission again. This will cache the task - // locally until a new actor creation notification is broadcast. We will - // attempt to reissue the actor creation tasks for all cached actor tasks - // in rerun_actor_creation_tasks_timeout_handler. - handle_actor_task_submitted(state, state->algorithm_state, *execution_spec); - } -} - -/** - * Give a task directly to another local scheduler. This is currently only used - * for assigning actor tasks to the local scheduler responsible for that actor. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param spec The task specification to schedule. - * @param local_scheduler_id The ID of the local scheduler to give the task to. - * @return Void. - */ -void give_task_to_local_scheduler(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec, - DBClientID local_scheduler_id) { - if (local_scheduler_id == get_db_client_id(state->db)) { - RAY_LOG(WARNING) << "Local scheduler is trying to assign a task to itself."; - } - RAY_CHECK(state->db != NULL); - /* Assign the task to the relevant local scheduler. */ - RAY_CHECK(state->config.global_scheduler_exists); - Task *task = - Task_alloc(execution_spec, TaskStatus::SCHEDULED, local_scheduler_id); - auto retryInfo = RetryInfo{ - .num_retries = 0, // This value is unused. - .timeout = 0, // This value is unused. - .fail_callback = give_task_to_local_scheduler_retry, - }; - - task_table_add_task(state->db, task, &retryInfo, NULL, state); -} - -void give_task_to_global_scheduler_retry(UniqueID id, - void *user_context, - void *user_data) { - LocalSchedulerState *state = (LocalSchedulerState *) user_context; - Task *task = (Task *) user_data; - RAY_CHECK(Task_state(task) == TaskStatus::WAITING); - - TaskExecutionSpec *execution_spec = Task_task_execution_spec(task); - TaskSpec *spec = execution_spec->Spec(); - RAY_CHECK(!TaskSpec_is_actor_task(spec)); - - give_task_to_global_scheduler(state, state->algorithm_state, *execution_spec); -} - -/** - * Give a task to the global scheduler to schedule. - * - * @param state The scheduler state. - * @param algorithm_state The scheduling algorithm state. - * @param spec The task specification to schedule. - * @return Void. - */ -void give_task_to_global_scheduler(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec) { - if (state->db == NULL || !state->config.global_scheduler_exists) { - /* A global scheduler is not available, so queue the task locally. */ - queue_task_locally(state, algorithm_state, execution_spec, false); - return; - } - /* Pass on the task to the global scheduler. */ - RAY_CHECK(state->config.global_scheduler_exists); - Task *task = Task_alloc(execution_spec, TaskStatus::WAITING, - get_db_client_id(state->db)); - RAY_CHECK(state->db != NULL); - auto retryInfo = RetryInfo{ - .num_retries = 0, // This value is unused. - .timeout = 0, // This value is unused. - .fail_callback = give_task_to_global_scheduler_retry, - }; - task_table_add_task(state->db, task, &retryInfo, NULL, state); -} - -bool resource_constraints_satisfied(LocalSchedulerState *state, - TaskSpec *spec) { - /* At the local scheduler, if required resource vector exceeds either static - * or dynamic resource vector, the resource constraint is not satisfied. */ - for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) { - double required_resource = resource_pair.second; - if (required_resource > state->static_resources[resource_pair.first] || - required_resource > state->dynamic_resources[resource_pair.first]) { - return false; - } - } - - if (TaskSpec_is_actor_creation_task(spec) && - state->static_resources["CPU"] != 0) { - return false; - } - - return true; -} - -void handle_task_submitted(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec) { - TaskSpec *spec = execution_spec.Spec(); - /* TODO(atumanov): if static is satisfied and local objects ready, but dynamic - * resource is currently unavailable, then consider queueing task locally and - * recheck dynamic next time. */ - - // If this task's constraints are satisfied, dependencies are available - // locally, and there is an available worker, then enqueue the task in the - // dispatch queue and trigger task dispatch. Otherwise, pass the task along to - // the global scheduler if there is one. - // Note that actor creation tasks automatically go to the global scheduler. - // See https://github.com/ray-project/ray/issues/1756 for more discussion. - // This is a hack to improve actor load balancing (and to prevent the scenario - // where all actors are started locally). - if (resource_constraints_satisfied(state, spec) && - (algorithm_state->available_workers.size() > 0) && - can_run(algorithm_state, execution_spec) && - !TaskSpec_is_actor_creation_task(spec)) { - queue_dispatch_task(state, algorithm_state, execution_spec, false); - } else { - /* Give the task to the global scheduler to schedule, if it exists. */ - give_task_to_global_scheduler(state, algorithm_state, execution_spec); - } - - /* Try to dispatch tasks, since we may have added one to the queue. */ - dispatch_tasks(state, algorithm_state); -} - -void handle_actor_task_submitted(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec) { - TaskSpec *task_spec = execution_spec.Spec(); - RAY_CHECK(TaskSpec_is_actor_task(task_spec)); - ActorID actor_id = TaskSpec_actor_id(task_spec); - - if (state->actor_mapping.count(actor_id) == 0) { - // Create a copy of the task to write to the task table. - Task *task = Task_alloc( - task_spec, execution_spec.SpecSize(), TaskStatus::ACTOR_CACHED, - get_db_client_id(state->db), execution_spec.ExecutionDependencies()); - - /* Add this task to a queue of tasks that have been submitted but the local - * scheduler doesn't know which actor is responsible for them. These tasks - * will be resubmitted (internally by the local scheduler) whenever a new - * actor notification arrives. NOTE(swang): These tasks have not yet been - * added to the task table. */ - TaskExecutionSpec task_entry = TaskExecutionSpec(&execution_spec); - algorithm_state->cached_submitted_actor_tasks.push_back( - std::move(task_entry)); - - // Even if the task can't be assigned to a worker yet, we should still write - // it to the task table. TODO(rkn): There's no need to do this more than - // once, and we could run into problems if we have very large numbers of - // tasks in this cache. - task_table_add_task(state->db, task, NULL, NULL, NULL); - - return; - } - - if (state->actor_mapping[actor_id].local_scheduler_id == - get_db_client_id(state->db)) { - /* This local scheduler is responsible for the actor, so handle the task - * locally. */ - queue_task_locally(state, algorithm_state, execution_spec, false); - /* Attempt to dispatch tasks to this actor. */ - dispatch_actor_task(state, algorithm_state, actor_id); - } else { - /* This local scheduler is not responsible for the task, so find the local - * scheduler that is responsible for this actor and assign the task directly - * to that local scheduler. */ - give_task_to_local_scheduler( - state, algorithm_state, execution_spec, - state->actor_mapping[actor_id].local_scheduler_id); - } -} - -void handle_actor_creation_notification( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id) { - int num_cached_actor_tasks = - algorithm_state->cached_submitted_actor_tasks.size(); - - for (int i = 0; i < num_cached_actor_tasks; ++i) { - TaskExecutionSpec &task = algorithm_state->cached_submitted_actor_tasks[i]; - /* Note that handle_actor_task_submitted may append the spec to the end of - * the cached_submitted_actor_tasks array. */ - handle_actor_task_submitted(state, algorithm_state, task); - } - /* Remove all the tasks that were resubmitted. This does not erase the tasks - * that were newly appended to the cached_submitted_actor_tasks array. */ - auto begin = algorithm_state->cached_submitted_actor_tasks.begin(); - algorithm_state->cached_submitted_actor_tasks.erase( - begin, begin + num_cached_actor_tasks); -} - -void handle_task_scheduled(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec) { - /* This callback handles tasks that were assigned to this local scheduler by - * the global scheduler, so we can safely assert that there is a connection to - * the database. */ - RAY_CHECK(state->db != NULL); - RAY_CHECK(state->config.global_scheduler_exists); - - // Currently, the global scheduler will never assign a task to a local - // scheduler that has 0 CPUs. - RAY_CHECK(state->static_resources["CPU"] != 0); - - // Push the task to the appropriate queue. - queue_task_locally(state, algorithm_state, execution_spec, true); - dispatch_tasks(state, algorithm_state); -} - -void handle_actor_task_scheduled(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec) { - TaskSpec *spec = execution_spec.Spec(); - /* This callback handles tasks that were assigned to this local scheduler by - * the global scheduler or by other workers, so we can safely assert that - * there is a connection to the database. */ - RAY_CHECK(state->db != NULL); - RAY_CHECK(state->config.global_scheduler_exists); - /* Check that the task is meant to run on an actor that this local scheduler - * is responsible for. */ - RAY_CHECK(TaskSpec_is_actor_task(spec)); - ActorID actor_id = TaskSpec_actor_id(spec); - if (state->actor_mapping.count(actor_id) == 1) { - RAY_CHECK(state->actor_mapping[actor_id].local_scheduler_id == - get_db_client_id(state->db)); - } else { - /* This means that an actor has been assigned to this local scheduler, and a - * task for that actor has been received by this local scheduler, but this - * local scheduler has not yet processed the notification about the actor - * creation. This may be possible though should be very uncommon. If it does - * happen, it's ok. */ - RAY_LOG(INFO) << "handle_actor_task_scheduled called on local scheduler " - << "but the corresponding actor_map_entry is not present. " - << "This should be rare."; - } - /* Push the task to the appropriate queue. */ - queue_task_locally(state, algorithm_state, execution_spec, true); - dispatch_actor_task(state, algorithm_state, actor_id); -} - -void handle_worker_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - RAY_CHECK(worker->task_in_progress == NULL); - /* Check that the worker isn't in the pool of available workers. */ - RAY_CHECK(!worker_in_vector(algorithm_state->available_workers, worker)); - - /* Check that the worker isn't in the list of blocked workers. */ - RAY_CHECK(!worker_in_vector(algorithm_state->blocked_workers, worker)); - - /* If the worker was executing a task, it must have finished, so remove it - * from the list of executing workers. If the worker is connecting for the - * first time, it will not be in the list of executing workers. */ - remove_worker_from_vector(algorithm_state->executing_workers, worker); - /* Double check that we successfully removed the worker. */ - RAY_CHECK(!worker_in_vector(algorithm_state->executing_workers, worker)); - - /* Add worker to the list of available workers. */ - algorithm_state->available_workers.push_back(worker); - - /* Try to dispatch tasks. */ - dispatch_all_tasks(state, algorithm_state); -} - -void handle_worker_removed(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - /* Make sure this is not an actor. */ - RAY_CHECK(worker->actor_id.is_nil()); - - /* Make sure that we remove the worker at most once. */ - int num_times_removed = 0; - - /* Remove the worker from available workers, if it's there. */ - bool removed_from_available = - remove_worker_from_vector(algorithm_state->available_workers, worker); - num_times_removed += removed_from_available; - /* Double check that we actually removed the worker. */ - RAY_CHECK(!worker_in_vector(algorithm_state->available_workers, worker)); - - /* Remove the worker from executing workers, if it's there. */ - bool removed_from_executing = - remove_worker_from_vector(algorithm_state->executing_workers, worker); - num_times_removed += removed_from_executing; - /* Double check that we actually removed the worker. */ - RAY_CHECK(!worker_in_vector(algorithm_state->executing_workers, worker)); - - /* Remove the worker from blocked workers, if it's there. */ - bool removed_from_blocked = - remove_worker_from_vector(algorithm_state->blocked_workers, worker); - num_times_removed += removed_from_blocked; - /* Double check that we actually removed the worker. */ - RAY_CHECK(!worker_in_vector(algorithm_state->blocked_workers, worker)); - - /* Make sure we removed the worker at most once. */ - RAY_CHECK(num_times_removed <= 1); - - /* Attempt to dispatch some tasks because some resources may have freed up. */ - dispatch_all_tasks(state, algorithm_state); -} - -void handle_actor_worker_disconnect(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker, - bool cleanup) { - /* Fail all in progress or queued tasks of the actor. */ - if (!cleanup) { - if (state->db != NULL) { - actor_table_mark_removed(state->db, worker->actor_id); - } - - if (worker->task_in_progress != NULL) { - finish_killed_task(state, - *Task_task_execution_spec(worker->task_in_progress)); - } - - state->removed_actors.insert(worker->actor_id); - - RAY_CHECK(algorithm_state->local_actor_infos.count(worker->actor_id) != 0); - LocalActorInfo &entry = - algorithm_state->local_actor_infos.find(worker->actor_id)->second; - for (auto &task : *entry.task_queue) { - finish_killed_task(state, task); - } - } - - remove_actor(algorithm_state, worker->actor_id); - - /* Attempt to dispatch some tasks because some resources may have freed up. */ - dispatch_all_tasks(state, algorithm_state); - - /* Start a worker to replace the removed actor's worker and replenish the - * worker pool. */ - start_worker(state); -} - -/* NOTE(swang): For tasks that saved a checkpoint, we should consider - * overwriting the result table entries for the current task frontier to - * avoid duplicate task submissions during reconstruction. */ -void handle_actor_worker_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - ActorID actor_id = worker->actor_id; - RAY_CHECK(!actor_id.is_nil()); - /* Get the actor info for this worker. */ - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) == 1); - LocalActorInfo &entry = - algorithm_state->local_actor_infos.find(actor_id)->second; - RAY_CHECK(worker == entry.worker); - RAY_CHECK(!entry.worker_available); - /* If an actor task was assigned, mark returned dummy object as locally - * available. This is not added to the object table, so the update will be - * invisible to other nodes. */ - /* NOTE(swang): These objects are never cleaned up. We should consider - * removing the objects, e.g., when an actor is terminated. */ - if (!entry.execution_dependency.is_nil()) { - handle_object_available(state, algorithm_state, entry.execution_dependency); - } - /* Unset the fields indicating an assigned task. */ - entry.worker_available = true; - /* Assign new tasks if possible. */ - dispatch_all_tasks(state, algorithm_state); -} - -void handle_worker_blocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - /* Find the worker in the list of executing workers. */ - RAY_CHECK( - remove_worker_from_vector(algorithm_state->executing_workers, worker)); - - /* Check that the worker isn't in the list of blocked workers. */ - RAY_CHECK(!worker_in_vector(algorithm_state->blocked_workers, worker)); - - /* Add the worker to the list of blocked workers. */ - algorithm_state->blocked_workers.push_back(worker); - - /* Try to dispatch tasks, since we may have freed up some resources. */ - dispatch_all_tasks(state, algorithm_state); -} - -void handle_actor_worker_blocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - /* The actor case doesn't use equivalents of the blocked_workers and - * executing_workers lists. Are these necessary? */ - /* Try to dispatch tasks, since we may have freed up some resources. */ - dispatch_all_tasks(state, algorithm_state); -} - -void handle_worker_unblocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) { - /* Find the worker in the list of blocked workers. */ - RAY_CHECK( - remove_worker_from_vector(algorithm_state->blocked_workers, worker)); - - /* Check that the worker isn't in the list of executing workers. */ - RAY_CHECK(!worker_in_vector(algorithm_state->executing_workers, worker)); - - /* Add the worker to the list of executing workers. */ - algorithm_state->executing_workers.push_back(worker); -} - -void handle_actor_worker_unblocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker) {} - -void handle_object_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ObjectID object_id) { - auto object_entry_it = algorithm_state->remote_objects.find(object_id); - - ObjectEntry entry; - /* Get the entry for this object from the active fetch request, or allocate - * one if needed. */ - if (object_entry_it != algorithm_state->remote_objects.end()) { - /* Remove the object from the active fetch requests. */ - entry = object_entry_it->second; - algorithm_state->remote_objects.erase(object_id); - } - - /* Add the entry to the set of locally available objects. */ - RAY_CHECK(algorithm_state->local_objects.count(object_id) == 0); - algorithm_state->local_objects[object_id] = entry; - - if (!entry.dependent_tasks.empty()) { - /* Out of the tasks that were dependent on this object, if they are now - * ready to run, move them to the dispatch queue. */ - for (auto &it : entry.dependent_tasks) { - if (can_run(algorithm_state, *it)) { - if (TaskSpec_is_actor_task(it->Spec())) { - insert_actor_task_queue(state, algorithm_state, std::move(*it)); - } else { - algorithm_state->dispatch_task_queue->push_back(std::move(*it)); - } - /* Remove the entry with a matching TaskSpec pointer from the waiting - * queue, but do not free the task spec. */ - algorithm_state->waiting_task_queue->erase(it); - } - } - /* Try to dispatch tasks, since we may have added some from the waiting - * queue. */ - dispatch_all_tasks(state, algorithm_state); - /* Clean up the records for dependent tasks. */ - entry.dependent_tasks.clear(); - } -} - -void handle_object_removed(LocalSchedulerState *state, - ObjectID removed_object_id) { - /* Remove the object from the set of locally available objects. */ - SchedulingAlgorithmState *algorithm_state = state->algorithm_state; - - RAY_CHECK(algorithm_state->local_objects.count(removed_object_id) == 1); - algorithm_state->local_objects.erase(removed_object_id); - - /* Track queued tasks that were dependent on this object. - * NOTE: Since objects often get removed in batches (e.g., during eviction), - * we may end up iterating through the queues many times in a row. If this - * turns out to be a bottleneck, consider tracking dependencies even for - * tasks in the dispatch queue, or batching object notifications. */ - /* Track the dependency for tasks that were in the dispatch queue. Remove - * these tasks from the dispatch queue and push them to the waiting queue. */ - for (auto it = algorithm_state->dispatch_task_queue->begin(); - it != algorithm_state->dispatch_task_queue->end();) { - if (it->DependsOn(removed_object_id)) { - /* This task was dependent on the removed object. */ - RAY_LOG(DEBUG) << "Moved task from dispatch queue back to waiting queue"; - algorithm_state->waiting_task_queue->push_back(std::move(*it)); - /* Remove the task from the dispatch queue, but do not free the task - * spec. */ - it = algorithm_state->dispatch_task_queue->erase(it); - } else { - /* The task can still run, so continue to the next task. */ - ++it; - } - } - - std::vector empty_actor_queues; - for (auto it = algorithm_state->actors_with_pending_tasks.begin(); - it != algorithm_state->actors_with_pending_tasks.end(); it++) { - auto actor_info = algorithm_state->local_actor_infos[*it]; - for (auto queue_it = actor_info.task_queue->begin(); - queue_it != actor_info.task_queue->end();) { - if (queue_it->DependsOn(removed_object_id)) { - /* This task was dependent on the removed object. */ - RAY_LOG(DEBUG) << "Moved task from actor dispatch queue back to " - << "waiting queue"; - algorithm_state->waiting_task_queue->push_back(std::move(*queue_it)); - /* Remove the task from the dispatch queue, but do not free the task - * spec. */ - queue_it = actor_info.task_queue->erase(queue_it); - if (actor_info.task_queue->size() == 0) { - empty_actor_queues.push_back(*it); - } - } else { - ++queue_it; - } - } - } - for (auto actor_id : empty_actor_queues) { - algorithm_state->actors_with_pending_tasks.erase(actor_id); - } - - /* Track the dependency for tasks that are in the waiting queue, including - * those that were just moved from the dispatch queue. */ - for (auto it = algorithm_state->waiting_task_queue->begin(); - it != algorithm_state->waiting_task_queue->end(); ++it) { - int64_t num_dependencies = it->NumDependencies(); - for (int64_t i = 0; i < num_dependencies; ++i) { - int count = it->DependencyIdCount(i); - for (int j = 0; j < count; ++j) { - ObjectID dependency_id = it->DependencyId(i, j); - if (dependency_id == removed_object_id) { - /* Do not request a transfer from other plasma managers if this is an - * execution dependency. */ - bool request_transfer = it->IsStaticDependency(i); - fetch_missing_dependency(state, algorithm_state, it, - removed_object_id.to_plasma_id(), - request_transfer); - } - } - } - } -} - -void handle_driver_removed(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - WorkerID driver_id) { - /* Loop over fetch requests. This must be done before we clean up the waiting - * task queue and the dispatch task queue because this map contains iterators - * for those lists, which will be invalidated when we clean up those lists.*/ - for (auto it = algorithm_state->remote_objects.begin(); - it != algorithm_state->remote_objects.end();) { - /* Loop over the tasks that are waiting for this object and remove the tasks - * for the removed driver. */ - auto task_it_it = it->second.dependent_tasks.begin(); - while (task_it_it != it->second.dependent_tasks.end()) { - /* If the dependent task was a task for the removed driver, remove it from - * this vector. */ - TaskSpec *spec = (*task_it_it)->Spec(); - if (TaskSpec_driver_id(spec) == driver_id) { - task_it_it = it->second.dependent_tasks.erase(task_it_it); - } else { - task_it_it++; - } - } - /* If there are no more dependent tasks for this object, then remove the - * ObjectEntry. */ - if (it->second.dependent_tasks.size() == 0) { - it = algorithm_state->remote_objects.erase(it); - } else { - it++; - } - } - - /* Remove this driver's tasks from the waiting task queue. */ - auto it = algorithm_state->waiting_task_queue->begin(); - while (it != algorithm_state->waiting_task_queue->end()) { - TaskSpec *spec = it->Spec(); - if (TaskSpec_driver_id(spec) == driver_id) { - it = algorithm_state->waiting_task_queue->erase(it); - } else { - it++; - } - } - - /* Remove this driver's tasks from the dispatch task queue. */ - it = algorithm_state->dispatch_task_queue->begin(); - while (it != algorithm_state->dispatch_task_queue->end()) { - TaskSpec *spec = it->Spec(); - if (TaskSpec_driver_id(spec) == driver_id) { - it = algorithm_state->dispatch_task_queue->erase(it); - } else { - it++; - } - } - - // Remove this driver's tasks from the cached actor tasks. Note that this loop - // could be very slow if the vector of cached actor tasks is very long. - for (auto it = algorithm_state->cached_submitted_actor_tasks.begin(); - it != algorithm_state->cached_submitted_actor_tasks.end();) { - TaskSpec *spec = (*it).Spec(); - if (TaskSpec_driver_id(spec) == driver_id) { - it = algorithm_state->cached_submitted_actor_tasks.erase(it); - } else { - ++it; - } - } - - /* TODO(rkn): Should we clean up the actor data structures? */ -} - -int num_waiting_tasks(SchedulingAlgorithmState *algorithm_state) { - return algorithm_state->waiting_task_queue->size(); -} - -int num_dispatch_tasks(SchedulingAlgorithmState *algorithm_state) { - return algorithm_state->dispatch_task_queue->size(); -} - -void print_worker_info(const char *message, - SchedulingAlgorithmState *algorithm_state) { - RAY_LOG(DEBUG) << message << ": " << algorithm_state->available_workers.size() - << " available, " << algorithm_state->executing_workers.size() - << " executing, " << algorithm_state->blocked_workers.size() - << " blocked"; -} - -std::unordered_map get_actor_task_counters( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id) { - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) != 0); - return algorithm_state->local_actor_infos[actor_id].task_counters; -} - -void set_actor_task_counters( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id, - const std::unordered_map &task_counters) { - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) != 0); - /* Overwrite the current task counters for the actor. This is necessary - * during reconstruction when resuming from a checkpoint so that we can - * resume the task frontier at the time that the checkpoint was saved. */ - auto &entry = algorithm_state->local_actor_infos[actor_id]; - entry.task_counters = task_counters; - - /* Filter out tasks for the actor that were submitted earlier than the new - * task counter. These represent tasks that executed before the actor's - * resumed checkpoint, and therefore should not be re-executed. */ - for (auto it = entry.task_queue->begin(); it != entry.task_queue->end();) { - /* Filter out duplicate tasks for the actor that are runnable. */ - TaskSpec *pending_task_spec = it->Spec(); - ActorHandleID handle_id = TaskSpec_actor_handle_id(pending_task_spec); - auto task_counter = entry.task_counters.find(handle_id); - if (task_counter != entry.task_counters.end() && - TaskSpec_actor_counter(pending_task_spec) < task_counter->second) { - /* If the task's counter is less than the highest count for that handle, - * then remove it from the actor's runnable queue. */ - it = entry.task_queue->erase(it); - } else { - it++; - } - } - for (auto it = algorithm_state->waiting_task_queue->begin(); - it != algorithm_state->waiting_task_queue->end();) { - /* Filter out duplicate tasks for the actor that are waiting on a missing - * dependency. */ - TaskSpec *spec = it->Spec(); - if (TaskSpec_actor_id(spec) == actor_id && - TaskSpec_actor_counter(spec) < - entry.task_counters[TaskSpec_actor_handle_id(spec)]) { - /* If the waiting task is for the same actor and its task counter is less - * than the highest count for that handle, then clear its object - * dependencies and remove it from the queue. */ - clear_missing_dependencies(algorithm_state, it); - it = algorithm_state->waiting_task_queue->erase(it); - } else { - it++; - } - } -} - -std::unordered_map get_actor_frontier( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id) { - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) != 0); - return algorithm_state->local_actor_infos[actor_id].frontier_dependencies; -} - -void set_actor_frontier( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id, - const std::unordered_map &frontier_dependencies) { - RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) != 0); - auto entry = algorithm_state->local_actor_infos[actor_id]; - entry.frontier_dependencies = frontier_dependencies; - for (auto frontier_dependency : entry.frontier_dependencies) { - if (algorithm_state->local_objects.count(frontier_dependency.second) == 0) { - handle_object_available(state, algorithm_state, - frontier_dependency.second); - } - } -} diff --git a/src/local_scheduler/local_scheduler_algorithm.h b/src/local_scheduler/local_scheduler_algorithm.h deleted file mode 100644 index 9238d5db5..000000000 --- a/src/local_scheduler/local_scheduler_algorithm.h +++ /dev/null @@ -1,438 +0,0 @@ -#ifndef LOCAL_SCHEDULER_ALGORITHM_H -#define LOCAL_SCHEDULER_ALGORITHM_H - -#include "local_scheduler_shared.h" -#include "common/task.h" -#include "state/local_scheduler_table.h" - -/* ==== The scheduling algorithm ==== - * - * This file contains declaration for all functions and data structures - * that need to be provided if you want to implement a new algorithms - * for the local scheduler. - * - */ - -/** - * Initialize the scheduler state. - * - * @return State managed by the scheduling algorithm. - */ -SchedulingAlgorithmState *SchedulingAlgorithmState_init(void); - -/** - * Free the scheduler state. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @return Void. - */ -void SchedulingAlgorithmState_free(SchedulingAlgorithmState *algorithm_state); - -/** - * - */ -void provide_scheduler_info(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerInfo *info); - -/** - * This function will be called when a new task is submitted by a worker for - * execution. The task will either be: - * 1. Put into the waiting queue, where it will wait for its dependencies to - * become available. - * 2. Put into the dispatch queue, where it will wait for an available worker. - * 3. Given to the global scheduler to be scheduled. - * - * Currently, the local scheduler policy is to keep the task if its - * dependencies are ready and there is an available worker. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param task Task that is submitted by the worker. - * @return Void. - */ -void handle_task_submitted(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec); - -/** - * This version of handle_task_submitted is used when the task being submitted - * is a method of an actor. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param task Task that is submitted by the worker. - * @return Void. - */ -void handle_actor_task_submitted(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec); - -/** - * This function will be called when the local scheduler receives a notification - * about the creation of a new actor. This can be used by the scheduling - * algorithm to resubmit cached actor tasks. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor being created. - * @return Void. - */ -void handle_actor_creation_notification( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id); - -/** - * This function will be called when a task is assigned by the global scheduler - * for execution on this local scheduler. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param task Task that is assigned by the global scheduler. - * @return Void. - */ -void handle_task_scheduled(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec); - -/** - * This function will be called when an actor task is assigned by the global - * scheduler or by another local scheduler for execution on this local - * scheduler. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param task Task that is assigned by the global scheduler. - * @return Void. - */ -void handle_actor_task_scheduled(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - TaskExecutionSpec &execution_spec); - -/** - * This function is called if a new object becomes available in the local - * plasma store. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param object_id ID of the object that became available. - * @return Void. - */ -void handle_object_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ObjectID object_id); - -/** - * This function is called if an object is removed from the local plasma store. - * - * @param state The state of the local scheduler. - * @param object_id ID of the object that was removed. - * @return Void. - */ -void handle_object_removed(LocalSchedulerState *state, ObjectID object_id); - -/** - * This function is called when a new worker becomes available. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is available. - * @return Void. - */ -void handle_worker_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * This function is called when a worker is removed. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is removed. - * @return Void. - */ -void handle_worker_removed(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * This version of handle_worker_available is called whenever the worker that is - * available is running an actor. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is available. - * @return Void. - */ -void handle_actor_worker_available(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * Handle the fact that a new worker is available for running an actor. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor running on the worker. - * @param initial_execution_dependency The dummy object ID of the actor - * creation task. - * @param worker The worker that was converted to an actor. - * @return Void. - */ -void handle_convert_worker_to_actor( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - const ActorID &actor_id, - const ObjectID &initial_execution_dependency, - LocalSchedulerClient *worker); - -/** - * Handle the fact that a worker running an actor has disconnected. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that was disconnected. - * @param cleanup Whether the disconnect was during cleanup. - * @return Void. - */ -void handle_actor_worker_disconnect(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker, - bool cleanup); - -/** - * This function is called when a worker that was executing a task becomes - * blocked on an object that isn't available locally yet. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is blocked. - * @return Void. - */ -void handle_worker_blocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * This function is called when an actor that was executing a task becomes - * blocked on an object that isn't available locally yet. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is blocked. - * @return Void. - */ -void handle_actor_worker_blocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * This function is called when a worker that was blocked on an object that - * wasn't available locally yet becomes unblocked. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is now unblocked. - * @return Void. - */ -void handle_worker_unblocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * This function is called when an actor that was blocked on an object that - * wasn't available locally yet becomes unblocked. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param worker The worker that is now unblocked. - * @return Void. - */ -void handle_actor_worker_unblocked(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - LocalSchedulerClient *worker); - -/** - * Process the fact that a driver has been removed. This will remove all of the - * tasks for that driver from the scheduling algorithm's internal data - * structures. - * - * @param state The state of the local scheduler. - * @param algorithm_state State maintained by the scheduling algorithm. - * @param driver_id The ID of the driver that was removed. - * @return Void. - */ -void handle_driver_removed(LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - WorkerID driver_id); - -/** - * This function fetches queued task's missing object dependencies. It is - * called every local_scheduler_fetch_timeout_milliseconds. - * - * @param loop The local scheduler's event loop. - * @param id The ID of the timer that triggers this function. - * @param context The function's context. - * @return An integer representing the time interval in seconds before the - * next invocation of the function. - */ -int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context); - -/** - * This function initiates reconstruction for task's missing object - * dependencies. It is called every - * local_scheduler_reconstruction_timeout_milliseconds, but it may not initiate - * reconstruction for every missing object. - * - * @param loop The local scheduler's event loop. - * @param id The ID of the timer that triggers this function. - * @param context The function's context. - * @return An integer representing the time interval in seconds before the - * next invocation of the function. - */ -int reconstruct_object_timeout_handler(event_loop *loop, - timer_id id, - void *context); - -/// This function initiates reconstruction for the actor creation tasks -/// corresponding to the actor tasks cached in the local scheduler. -/// -/// \param loop The local scheduler's event loop. -/// \param id The ID of the timer that triggers this function. -/// \param context The function's context. -/// \return An integer representing the time interval in seconds before the -/// next invocation of the function. -int rerun_actor_creation_tasks_timeout_handler(event_loop *loop, - timer_id id, - void *context); - -/** - * Check whether an object, including actor dummy objects, is locally - * available. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @param object_id The ID of the object to check for. - * @return A bool representing whether the object is locally available. - */ -bool object_locally_available(SchedulingAlgorithmState *algorithm_state, - ObjectID object_id); - -/// Spill some tasks back to the global scheduler. This function implements the -/// spillback policy. -/// -/// @param state The scheduler state. -/// @return Void. -void spillback_tasks_handler(LocalSchedulerState *state); - -/** - * A helper function to print debug information about the current state and - * number of workers. - * - * @param message A message to identify the log message. - * @param algorithm_state State maintained by the scheduling algorithm. - * @return Void. - */ -void print_worker_info(const char *message, - SchedulingAlgorithmState *algorithm_state); - -/* - * The actor frontier consists of the number of tasks executed so far and the - * execution dependencies required by the current runnable tasks, according to - * the actor's local scheduler. Since an actor may have multiple handles, the - * tasks submitted to the actor form a DAG, where nodes are tasks and edges are - * execution dependencies. The frontier is a cut across this DAG. The number of - * tasks so far is the number of nodes included in the DAG root's partition. - * - * The actor gets the current frontier of tasks from the local scheduler during - * a checkpoint save, so that it can save the point in the actor's lifetime at - * which the checkpoint was taken. If the actor later resumes from that - * checkpoint, the actor can set the current frontier of tasks in the local - * scheduler so that the same frontier of tasks can be made runnable again - * during reconstruction, and so that we do not duplicate execution of tasks - * that already executed before the checkpoint. - */ - -/** - * Get the number of tasks, per actor handle, that have been executed on an - * actor so far. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor whose task counters are returned. - * @return A map from handle ID to the number of tasks submitted by that handle - * that have executed so far. - */ -std::unordered_map get_actor_task_counters( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id); - -/** - * Set the number of tasks, per actor handle, that have been executed on an - * actor so far. All previous counts will be overwritten. Tasks that are - * waiting or runnable on the local scheduler that have a lower task count will - * be discarded, so that we don't duplicate execution. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor whose task counters are returned. - * @param task_counters A map from handle ID to the number of tasks submitted - * by that handle that have executed so far. - * @return Void. - */ -void set_actor_task_counters( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id, - const std::unordered_map &task_counters); - -/** - * Get the actor's frontier of task dependencies. - * NOTE(swang): The returned frontier only includes handles known by the local - * scheduler. It does not include handles for which the local scheduler has not - * seen a runnable task yet. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor whose task counters are returned. - * @return A map from handle ID to execution dependency for the earliest - * runnable task submitted through that handle. - */ -std::unordered_map get_actor_frontier( - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id); - -/** - * Set the actor's frontier of task dependencies. The previous frontier will be - * overwritten. Any tasks that have an execution dependency on the new frontier - * (and that have all other dependencies fulfilled) will become runnable. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @param actor_id The ID of the actor whose task counters are returned. - * @param frontier_dependencies A map from handle ID to execution dependency - * for the earliest runnable task submitted through that handle. - * @return Void. - */ -void set_actor_frontier( - LocalSchedulerState *state, - SchedulingAlgorithmState *algorithm_state, - ActorID actor_id, - const std::unordered_map &frontier_dependencies); - -/** The following methods are for testing purposes only. */ -#ifdef LOCAL_SCHEDULER_TEST -/** - * Get the number of tasks currently waiting for object dependencies to become - * available locally. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @return The number of tasks queued. - */ -int num_waiting_tasks(SchedulingAlgorithmState *algorithm_state); - -/** - * Get the number of tasks currently waiting for a worker to become available. - * - * @param algorithm_state State maintained by the scheduling algorithm. - * @return The number of tasks queued. - */ -int num_dispatch_tasks(SchedulingAlgorithmState *algorithm_state); -#endif - -#endif /* LOCAL_SCHEDULER_ALGORITHM_H */ diff --git a/src/local_scheduler/local_scheduler_client.cc b/src/local_scheduler/local_scheduler_client.cc deleted file mode 100644 index 09bda7f5b..000000000 --- a/src/local_scheduler/local_scheduler_client.cc +++ /dev/null @@ -1,385 +0,0 @@ -#include "local_scheduler_client.h" - -#include "common_protocol.h" -#include "format/local_scheduler_generated.h" -#include "ray/raylet/format/node_manager_generated.h" - -#include "common/io.h" -#include "common/task.h" -#include -#include -#include - -using MessageType = ray::local_scheduler::protocol::MessageType; - -LocalSchedulerConnection *LocalSchedulerConnection_init( - const char *local_scheduler_socket, - const UniqueID &client_id, - bool is_worker, - const JobID &driver_id, - bool use_raylet, - const Language &language) { - LocalSchedulerConnection *result = new LocalSchedulerConnection(); - result->use_raylet = use_raylet; - result->conn = connect_ipc_sock_retry(local_scheduler_socket, -1, -1); - - /* Register with the local scheduler. - * NOTE(swang): If the local scheduler exits and we are registered as a - * worker, we will get killed. */ - flatbuffers::FlatBufferBuilder fbb; - if (use_raylet) { - auto message = ray::protocol::CreateRegisterClientRequest( - fbb, is_worker, to_flatbuf(fbb, client_id), getpid(), - to_flatbuf(fbb, driver_id), language); - fbb.Finish(message); - } else { - auto message = ray::local_scheduler::protocol::CreateRegisterClientRequest( - fbb, is_worker, to_flatbuf(fbb, client_id), getpid(), - to_flatbuf(fbb, driver_id)); - fbb.Finish(message); - } - /* Register the process ID with the local scheduler. */ - int success = write_message( - result->conn, static_cast(MessageType::RegisterClientRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &result->write_mutex); - RAY_CHECK(success == 0) << "Unable to register worker with local scheduler"; - - return result; -} - -void LocalSchedulerConnection_free(LocalSchedulerConnection *conn) { - close(conn->conn); - delete conn; -} - -void local_scheduler_disconnect_client(LocalSchedulerConnection *conn) { - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::local_scheduler::protocol::CreateDisconnectClient(fbb); - fbb.Finish(message); - if (conn->use_raylet) { - write_message(conn->conn, static_cast( - MessageType::IntentionalDisconnectClient), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - } else { - write_message(conn->conn, - static_cast(MessageType::DisconnectClient), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - } -} - -void local_scheduler_log_event(LocalSchedulerConnection *conn, - uint8_t *key, - int64_t key_length, - uint8_t *value, - int64_t value_length, - double timestamp) { - flatbuffers::FlatBufferBuilder fbb; - auto key_string = fbb.CreateString((char *) key, key_length); - auto value_string = fbb.CreateString((char *) value, value_length); - auto message = ray::local_scheduler::protocol::CreateEventLogMessage( - fbb, key_string, value_string, timestamp); - fbb.Finish(message); - write_message(conn->conn, static_cast(MessageType::EventLogMessage), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -void local_scheduler_submit(LocalSchedulerConnection *conn, - const TaskExecutionSpec &execution_spec) { - flatbuffers::FlatBufferBuilder fbb; - auto execution_dependencies = - to_flatbuf(fbb, execution_spec.ExecutionDependencies()); - auto task_spec = - fbb.CreateString(reinterpret_cast(execution_spec.Spec()), - execution_spec.SpecSize()); - auto message = ray::local_scheduler::protocol::CreateSubmitTaskRequest( - fbb, execution_dependencies, task_spec); - fbb.Finish(message); - write_message(conn->conn, static_cast(MessageType::SubmitTask), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -void local_scheduler_submit_raylet( - LocalSchedulerConnection *conn, - const std::vector &execution_dependencies, - const ray::raylet::TaskSpecification &task_spec) { - flatbuffers::FlatBufferBuilder fbb; - auto execution_dependencies_message = to_flatbuf(fbb, execution_dependencies); - auto message = ray::local_scheduler::protocol::CreateSubmitTaskRequest( - fbb, execution_dependencies_message, task_spec.ToFlatbuffer(fbb)); - fbb.Finish(message); - write_message(conn->conn, static_cast(MessageType::SubmitTask), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -TaskSpec *local_scheduler_get_task(LocalSchedulerConnection *conn, - int64_t *task_size) { - int64_t type; - int64_t reply_size; - uint8_t *reply; - { - std::unique_lock guard(conn->mutex); - write_message(conn->conn, static_cast(MessageType::GetTask), 0, - NULL, &conn->write_mutex); - /* Receive a task from the local scheduler. This will block until the local - * scheduler gives this client a task. */ - read_message(conn->conn, &type, &reply_size, &reply); - } - if (type == static_cast(CommonMessageType::DISCONNECT_CLIENT)) { - RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection."; - exit(1); - } - RAY_CHECK(static_cast(type) == MessageType::ExecuteTask); - - /* Parse the flatbuffer object. */ - auto reply_message = - flatbuffers::GetRoot(reply); - - /* Create a copy of the task spec so we can free the reply. */ - *task_size = reply_message->task_spec()->size(); - TaskSpec *data = (TaskSpec *) reply_message->task_spec()->data(); - TaskSpec *spec = TaskSpec_copy(data, *task_size); - - // Set the GPU IDs for this task. We only do this for non-actor tasks because - // for actors the GPUs are associated with the actor itself and not with the - // actor methods. Note that this also processes GPUs for actor creation tasks. - if (!TaskSpec_is_actor_task(spec)) { - conn->gpu_ids.clear(); - for (size_t i = 0; i < reply_message->gpu_ids()->size(); ++i) { - conn->gpu_ids.push_back(reply_message->gpu_ids()->Get(i)); - } - } - - /* Free the original message from the local scheduler. */ - free(reply); - /* Return the copy of the task spec and pass ownership to the caller. */ - return spec; -} - -// This is temporarily duplicated from local_scheduler_get_task while we have -// the raylet and non-raylet code paths. -TaskSpec *local_scheduler_get_task_raylet(LocalSchedulerConnection *conn, - int64_t *task_size) { - int64_t type; - int64_t reply_size; - uint8_t *reply; - { - std::unique_lock guard(conn->mutex); - write_message(conn->conn, static_cast(MessageType::GetTask), 0, - NULL, &conn->write_mutex); - // Receive a task from the local scheduler. This will block until the local - // scheduler gives this client a task. - read_message(conn->conn, &type, &reply_size, &reply); - } - if (type == static_cast(CommonMessageType::DISCONNECT_CLIENT)) { - RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection."; - exit(1); - } - RAY_CHECK(type == static_cast(MessageType::ExecuteTask)); - - // Parse the flatbuffer object. - auto reply_message = flatbuffers::GetRoot(reply); - - // Create a copy of the task spec so we can free the reply. - *task_size = reply_message->task_spec()->size(); - const TaskSpec *data = - reinterpret_cast(reply_message->task_spec()->data()); - TaskSpec *spec = TaskSpec_copy(const_cast(data), *task_size); - - // Set the resource IDs for this task. - conn->resource_ids_.clear(); - for (size_t i = 0; i < reply_message->fractional_resource_ids()->size(); - ++i) { - auto const &fractional_resource_ids = - reply_message->fractional_resource_ids()->Get(i); - auto &acquired_resources = conn->resource_ids_[string_from_flatbuf( - *fractional_resource_ids->resource_name())]; - - size_t num_resource_ids = fractional_resource_ids->resource_ids()->size(); - size_t num_resource_fractions = - fractional_resource_ids->resource_fractions()->size(); - RAY_CHECK(num_resource_ids == num_resource_fractions); - RAY_CHECK(num_resource_ids > 0); - for (size_t j = 0; j < num_resource_ids; ++j) { - int64_t resource_id = fractional_resource_ids->resource_ids()->Get(j); - double resource_fraction = - fractional_resource_ids->resource_fractions()->Get(j); - if (num_resource_ids > 1) { - int64_t whole_fraction = resource_fraction; - RAY_CHECK(whole_fraction == resource_fraction); - } - acquired_resources.push_back( - std::make_pair(resource_id, resource_fraction)); - } - } - - // Free the original message from the local scheduler. - free(reply); - // Return the copy of the task spec and pass ownership to the caller. - return spec; -} - -void local_scheduler_task_done(LocalSchedulerConnection *conn) { - write_message(conn->conn, static_cast(MessageType::TaskDone), 0, - NULL, &conn->write_mutex); -} - -void local_scheduler_reconstruct_objects( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - bool fetch_only) { - flatbuffers::FlatBufferBuilder fbb; - auto object_ids_message = to_flatbuf(fbb, object_ids); - auto message = ray::local_scheduler::protocol::CreateReconstructObjects( - fbb, object_ids_message, fetch_only); - fbb.Finish(message); - write_message(conn->conn, - static_cast(MessageType::ReconstructObjects), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - /* TODO(swang): Propagate the error. */ -} - -void local_scheduler_log_message(LocalSchedulerConnection *conn) { - write_message(conn->conn, static_cast(MessageType::EventLogMessage), - 0, NULL, &conn->write_mutex); -} - -void local_scheduler_notify_unblocked(LocalSchedulerConnection *conn) { - write_message(conn->conn, static_cast(MessageType::NotifyUnblocked), - 0, NULL, &conn->write_mutex); -} - -void local_scheduler_put_object(LocalSchedulerConnection *conn, - TaskID task_id, - ObjectID object_id) { - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::local_scheduler::protocol::CreatePutObject( - fbb, to_flatbuf(fbb, task_id), to_flatbuf(fbb, object_id)); - fbb.Finish(message); - - write_message(conn->conn, static_cast(MessageType::PutObject), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -const std::vector local_scheduler_get_actor_frontier( - LocalSchedulerConnection *conn, - ActorID actor_id) { - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::local_scheduler::protocol::CreateGetActorFrontierRequest( - fbb, to_flatbuf(fbb, actor_id)); - fbb.Finish(message); - int64_t type; - std::vector reply; - { - std::unique_lock guard(conn->mutex); - write_message(conn->conn, - static_cast(MessageType::GetActorFrontierRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - - read_vector(conn->conn, &type, reply); - } - if (static_cast(type) == - CommonMessageType::DISCONNECT_CLIENT) { - RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection."; - exit(1); - } - RAY_CHECK(static_cast(type) == - MessageType::GetActorFrontierReply); - return reply; -} - -void local_scheduler_set_actor_frontier(LocalSchedulerConnection *conn, - const std::vector &frontier) { - write_message(conn->conn, static_cast(MessageType::SetActorFrontier), - frontier.size(), const_cast(frontier.data()), - &conn->write_mutex); -} - -std::pair, std::vector> local_scheduler_wait( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - int num_returns, - int64_t timeout_milliseconds, - bool wait_local) { - // Write request. - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::protocol::CreateWaitRequest( - fbb, to_flatbuf(fbb, object_ids), num_returns, timeout_milliseconds, - wait_local); - fbb.Finish(message); - int64_t type; - int64_t reply_size; - uint8_t *reply; - { - std::unique_lock guard(conn->mutex); - write_message(conn->conn, - static_cast(ray::protocol::MessageType::WaitRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - // Read result. - read_message(conn->conn, &type, &reply_size, &reply); - } - RAY_CHECK(static_cast(type) == - ray::protocol::MessageType::WaitReply); - auto reply_message = flatbuffers::GetRoot(reply); - // Convert result. - std::pair, std::vector> result; - auto found = reply_message->found(); - for (uint i = 0; i < found->size(); i++) { - ObjectID object_id = ObjectID::from_binary(found->Get(i)->str()); - result.first.push_back(object_id); - } - auto remaining = reply_message->remaining(); - for (uint i = 0; i < remaining->size(); i++) { - ObjectID object_id = ObjectID::from_binary(remaining->Get(i)->str()); - result.second.push_back(object_id); - } - /* Free the original message from the local scheduler. */ - free(reply); - return result; -} - -void local_scheduler_push_error(LocalSchedulerConnection *conn, - const JobID &job_id, - const std::string &type, - const std::string &error_message, - double timestamp) { - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::protocol::CreatePushErrorRequest( - fbb, to_flatbuf(fbb, job_id), fbb.CreateString(type), - fbb.CreateString(error_message), timestamp); - fbb.Finish(message); - - write_message(conn->conn, static_cast( - ray::protocol::MessageType::PushErrorRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -void local_scheduler_push_profile_events( - LocalSchedulerConnection *conn, - const ProfileTableDataT &profile_events) { - flatbuffers::FlatBufferBuilder fbb; - - auto message = CreateProfileTableData(fbb, &profile_events); - fbb.Finish(message); - - write_message(conn->conn, - static_cast( - ray::protocol::MessageType::PushProfileEventsRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); -} - -void local_scheduler_free_objects_in_object_store( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - bool local_only) { - flatbuffers::FlatBufferBuilder fbb; - auto message = ray::protocol::CreateFreeObjectsRequest( - fbb, local_only, to_flatbuf(fbb, object_ids)); - fbb.Finish(message); - - int success = write_message( - conn->conn, - static_cast( - ray::protocol::MessageType::FreeObjectsInObjectStoreRequest), - fbb.GetSize(), fbb.GetBufferPointer(), &conn->write_mutex); - RAY_CHECK(success == 0) << "Failed to write message to raylet."; -} diff --git a/src/local_scheduler/local_scheduler_client.h b/src/local_scheduler/local_scheduler_client.h deleted file mode 100644 index bb4fdb345..000000000 --- a/src/local_scheduler/local_scheduler_client.h +++ /dev/null @@ -1,260 +0,0 @@ -#ifndef LOCAL_SCHEDULER_CLIENT_H -#define LOCAL_SCHEDULER_CLIENT_H - -#include - -#include "common/task.h" -#include "local_scheduler_shared.h" -#include "ray/raylet/task_spec.h" - -struct LocalSchedulerConnection { - /// True if we should use the raylet code path and false otherwise. - bool use_raylet; - /** File descriptor of the Unix domain socket that connects to local - * scheduler. */ - int conn; - /** The IDs of the GPUs that this client can use. NOTE(rkn): This is only used - * by legacy Ray and will be deprecated. */ - std::vector gpu_ids; - /// A map from resource name to the resource IDs that are currently reserved - /// for this worker. Each pair consists of the resource ID and the fraction - /// of that resource allocated for this worker. - std::unordered_map>> - resource_ids_; - /// A mutex to protect stateful operations of the local scheduler client. - std::mutex mutex; - /// A mutext to protect write operations of the local scheduler client. - std::mutex write_mutex; -}; - -/** - * Connect to the local scheduler. - * - * @param local_scheduler_socket The name of the socket to use to connect to the - * local scheduler. - * @param worker_id A unique ID to represent the worker. - * @param is_worker Whether this client is a worker. If it is a worker, an - * additional message will be sent to register as one. - * @param driver_id The ID of the driver. This is non-nil if the client is a - * driver. - * @param use_raylet True if we should use the raylet code path and false - * otherwise. - * @return The connection information. - */ -LocalSchedulerConnection *LocalSchedulerConnection_init( - const char *local_scheduler_socket, - const UniqueID &worker_id, - bool is_worker, - const JobID &driver_id, - bool use_raylet, - const Language &language); - -/** - * Disconnect from the local scheduler. - * - * @param conn Local scheduler connection information returned by - * LocalSchedulerConnection_init. - * @return Void. - */ -void LocalSchedulerConnection_free(LocalSchedulerConnection *conn); - -/** - * Submit a task to the local scheduler. - * - * @param conn The connection information. - * @param execution_spec The execution spec for the task to submit. - * @return Void. - */ -void local_scheduler_submit(LocalSchedulerConnection *conn, - const TaskExecutionSpec &execution_spec); - -/// Submit a task using the raylet code path. -/// -/// \param The connection information. -/// \param The execution dependencies. -/// \param The task specification. -/// \return Void. -void local_scheduler_submit_raylet( - LocalSchedulerConnection *conn, - const std::vector &execution_dependencies, - const ray::raylet::TaskSpecification &task_spec); - -/** - * Notify the local scheduler that this client is disconnecting gracefully. This - * is used by actors to exit gracefully so that the local scheduler doesn't - * propagate an error message to the driver. - * - * @param conn The connection information. - * @return Void. - */ -void local_scheduler_disconnect_client(LocalSchedulerConnection *conn); - -/** - * Log an event to the event log. This will call RPUSH key value. We use RPUSH - * instead of SET so that it is possible to flush the log multiple times with - * the same key (for example the key might be shared across logging calls in the - * same task on a worker). - * - * @param conn The connection information. - * @param key The key to store the event in. - * @param key_length The length of the key. - * @param value The value to store. - * @param value_length The length of the value. - * @param timestamp The time that the event is logged. - * @return Void. - */ -void local_scheduler_log_event(LocalSchedulerConnection *conn, - uint8_t *key, - int64_t key_length, - uint8_t *value, - int64_t value_length, - double timestamp); - -/** - * Get next task for this client. This will block until the scheduler assigns - * a task to this worker. This allocates and returns a task, and so the task - * must be freed by the caller. - * - * @todo When does this actually get freed? - * - * @param conn The connection information. - * @param task_size A pointer to fill out with the task size. - * @return The address of the assigned task. - */ -TaskSpec *local_scheduler_get_task(LocalSchedulerConnection *conn, - int64_t *task_size); - -/// Get next task for this client. This will block until the scheduler assigns -/// a task to this worker. This allocates and returns a task, and so the task -/// must be freed by the caller. -/// -/// \param conn The connection information. -/// \param task_size A pointer to fill out with the task size. -/// \return The address of the assigned task. -TaskSpec *local_scheduler_get_task_raylet(LocalSchedulerConnection *conn, - int64_t *task_size); - -/** - * Tell the local scheduler that the client has finished executing a task. - * - * @param conn The connection information. - * @return Void. - */ -void local_scheduler_task_done(LocalSchedulerConnection *conn); - -/** - * Tell the local scheduler to reconstruct or fetch objects. - * - * @param conn The connection information. - * @param object_ids The IDs of the objects to reconstruct. - * @param fetch_only Only fetch objects, do not reconstruct them. - * @return Void. - */ -void local_scheduler_reconstruct_objects( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - bool fetch_only = false); - -/** - * Send a log message to the local scheduler. - * - * @param conn The connection information. - * @return Void. - */ -void local_scheduler_log_message(LocalSchedulerConnection *conn); - -/** - * Notify the local scheduler that this client (worker) is no longer blocked. - * - * @param conn The connection information. - * @return Void. - */ -void local_scheduler_notify_unblocked(LocalSchedulerConnection *conn); - -/** - * Record the mapping from object ID to task ID for put events. - * - * @param conn The connection information. - * @param task_id The ID of the task that called put. - * @param object_id The ID of the object being stored. - * @return Void. - */ -void local_scheduler_put_object(LocalSchedulerConnection *conn, - TaskID task_id, - ObjectID object_id); - -/** - * Get an actor's current task frontier. - * - * @param conn The connection information. - * @param actor_id The ID of the actor whose frontier is returned. - * @return A byte vector that can be traversed as an ActorFrontier flatbuffer. - */ -const std::vector local_scheduler_get_actor_frontier( - LocalSchedulerConnection *conn, - ActorID actor_id); - -/** - * Set an actor's current task frontier. - * - * @param conn The connection information. - * @param frontier An ActorFrontier flatbuffer to set the frontier to. - * @return Void. - */ -void local_scheduler_set_actor_frontier(LocalSchedulerConnection *conn, - const std::vector &frontier); - -/// Wait for the given objects until timeout expires or num_return objects are -/// found. -/// -/// \param conn The connection information. -/// \param object_ids The objects to wait for. -/// \param num_returns The number of objects to wait for. -/// \param timeout_milliseconds Duration, in milliseconds, to wait before -/// returning. -/// \param wait_local Whether to wait for objects to appear on this node. -/// \return A pair with the first element containing the object ids that were -/// found, and the second element the objects that were not found. -std::pair, std::vector> local_scheduler_wait( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - int num_returns, - int64_t timeout_milliseconds, - bool wait_local); - -/// Push an error to the relevant driver. -/// -/// \param conn The connection information. -/// \param The ID of the job that the error is for. -/// \param The type of the error. -/// \param The error message. -/// \param The timestamp of the error. -/// \return Void. -void local_scheduler_push_error(LocalSchedulerConnection *conn, - const JobID &job_id, - const std::string &type, - const std::string &error_message, - double timestamp); - -/// Store some profile events in the GCS. -/// -/// \param conn The connection information. -/// \param profile_events A batch of profiling event information. -/// \return Void. -void local_scheduler_push_profile_events( - LocalSchedulerConnection *conn, - const ProfileTableDataT &profile_events); - -/// Free a list of objects from object stores. -/// -/// \param conn The connection information. -/// \param object_ids A list of ObjectsIDs to be deleted. -/// \param local_only Whether keep this request with local object store -/// or send it to all the object stores. -/// \return Void. -void local_scheduler_free_objects_in_object_store( - LocalSchedulerConnection *conn, - const std::vector &object_ids, - bool local_only); - -#endif diff --git a/src/local_scheduler/local_scheduler_shared.h b/src/local_scheduler/local_scheduler_shared.h deleted file mode 100644 index 572f14a6f..000000000 --- a/src/local_scheduler/local_scheduler_shared.h +++ /dev/null @@ -1,137 +0,0 @@ -#ifndef LOCAL_SCHEDULER_SHARED_H -#define LOCAL_SCHEDULER_SHARED_H - -#include "common/task.h" -#include "common/state/table.h" -#include "common/state/db.h" -#include "plasma/client.h" -#include "ray/gcs/client.h" - -#include -#include -#include -#include - -/** This struct is used to maintain a mapping from actor IDs to the ID of the - * local scheduler that is responsible for the actor. */ -struct ActorMapEntry { - /** The ID of the driver that created the actor. */ - WorkerID driver_id; - /** The ID of the local scheduler that is responsible for the actor. */ - DBClientID local_scheduler_id; -}; - -/** Internal state of the scheduling algorithm. */ -typedef struct SchedulingAlgorithmState SchedulingAlgorithmState; - -struct LocalSchedulerClient; - -/** A struct storing the configuration state of the local scheduler. This should - * consist of values that don't change over the lifetime of the local - * scheduler. */ -typedef struct { - /** The script to use when starting a new worker. */ - const char **start_worker_command; - /** Whether there is a global scheduler. */ - bool global_scheduler_exists; -} local_scheduler_config; - -/** The state of the local scheduler. */ -struct LocalSchedulerState { - /** The configuration for the local scheduler. */ - local_scheduler_config config; - /** The local scheduler event loop. */ - event_loop *loop; - /** List of workers available to this node. This is used to free the worker - * structs when we free the scheduler state and also to access the worker - * structs in the tests. */ - std::list workers; - /** A set of driver IDs corresponding to drivers that have been removed. This - * is used to make sure we don't execute any tasks belong to dead drivers. */ - std::unordered_set removed_drivers; - /** A set of actors IDs corresponding to local actors that have been removed. - * This ensures we can reject any tasks destined for dead actors. */ - std::unordered_set removed_actors; - /** List of the process IDs for child processes (workers) started by the - * local scheduler that have not sent a REGISTER_PID message yet. */ - std::vector child_pids; - /** A hash table mapping actor IDs to the db_client_id of the local scheduler - * that is responsible for the actor. */ - std::unordered_map actor_mapping; - /** The handle to the database. */ - DBHandle *db; - /** The Plasma client. */ - plasma::PlasmaClient *plasma_conn; - /** State for the scheduling algorithm. */ - SchedulingAlgorithmState *algorithm_state; - /** Input buffer, used for reading input in process_message to avoid - * allocation for each call to process_message. */ - std::vector input_buffer; - /** Vector of static attributes associated with the node owned by this local - * scheduler. */ - std::unordered_map static_resources; - /** Vector of dynamic attributes associated with the node owned by this local - * scheduler. */ - std::unordered_map dynamic_resources; - /** The IDs of the available GPUs. There is redundancy here in that - * available_gpus.size() == dynamic_resources[ResourceIndex_GPU] should - * always be true. */ - std::vector available_gpus; - /** The time (in milliseconds since the Unix epoch) when the most recent - * heartbeat was sent. */ - int64_t previous_heartbeat_time; -}; - -/** Contains all information associated with a local scheduler client. */ -struct LocalSchedulerClient { - /** The socket used to communicate with the client. */ - int sock; - /** True if the client has registered and false otherwise. */ - bool registered; - /** True if the client has sent a disconnect message to the local scheduler - * and false otherwise. If this is true, then the local scheduler will not - * propagate an error message to the driver when the client exits. */ - bool disconnected; - /** True if the client is a worker and false if it is a driver. */ - bool is_worker; - /** The worker ID if the client is a worker and the driver ID if the client is - * a driver. */ - WorkerID client_id; - /** A pointer to the task object that is currently running on this client. If - * no task is running on the worker, this will be NULL. This is used to - * update the task table. */ - Task *task_in_progress; - /** An array of resource counts currently in use by the worker. */ - std::unordered_map resources_in_use; - /** A vector of the IDs of the GPUs that the worker is currently using. If the - * worker is an actor, this will be constant throughout the lifetime of the - * actor (and will be equal to the number of GPUs requested by the actor). If - * the worker is not an actor, this will be constant for the duration of a - * task and will have length equal to the number of GPUs requested by the - * task (in particular it will not change if the task blocks). */ - std::vector gpus_in_use; - /** A flag to indicate whether this worker is currently blocking on an - * object(s) that isn't available locally yet. */ - bool is_blocked; - /** The process ID of the client. If this is set to zero, the client has not - * yet registered a process ID. */ - pid_t pid; - /** Whether the client is a child process of the local scheduler. */ - bool is_child; - /** The ID of the actor on this worker. If there is no actor running on this - * worker, this should be NIL_ACTOR_ID. */ - ActorID actor_id; - /** A pointer to the local scheduler state. */ - LocalSchedulerState *local_scheduler_state; -}; - -/** - * Free the local scheduler state. This disconnects all clients and notifies - * the global scheduler of the local scheduler's exit. - * - * @param state The state to free. - * @return Void - */ -void LocalSchedulerState_free(LocalSchedulerState *state); - -#endif /* LOCAL_SCHEDULER_SHARED_H */ diff --git a/src/local_scheduler/test/local_scheduler_tests.cc b/src/local_scheduler/test/local_scheduler_tests.cc deleted file mode 100644 index b155ea949..000000000 --- a/src/local_scheduler/test/local_scheduler_tests.cc +++ /dev/null @@ -1,704 +0,0 @@ -#include "greatest.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "common.h" -#include "test/test_common.h" -#include "test/example_task.h" -#include "event_loop.h" -#include "io.h" -#include "task.h" -#include "state/object_table.h" -#include "state/task_table.h" -#include "state/redis.h" - -#include "local_scheduler_shared.h" -#include "local_scheduler.h" -#include "local_scheduler_algorithm.h" -#include "local_scheduler_client.h" - -SUITE(local_scheduler_tests); - -TaskBuilder *g_task_builder = NULL; - -const char *plasma_store_socket_name = "/tmp/plasma_store_socket_1"; -const char *plasma_manager_socket_name_format = "/tmp/plasma_manager_socket_%d"; -const char *local_scheduler_socket_name_format = - "/tmp/local_scheduler_socket_%d"; - -int64_t timeout_handler(event_loop *loop, int64_t id, void *context) { - event_loop_stop(loop); - return EVENT_LOOP_TIMER_DONE; -} - -typedef struct { - /** A socket to mock the Plasma manager. Clients (such as workers) that - * connect to this file descriptor must be accepted. */ - int plasma_manager_fd; - /** A socket to communicate with the Plasma store. */ - int plasma_store_fd; - /** Local scheduler's socket for IPC requests. */ - int local_scheduler_fd; - /** Local scheduler's local scheduler state. */ - LocalSchedulerState *local_scheduler_state; - /** Local scheduler's event loop. */ - event_loop *loop; - /** Number of local scheduler client connections, or mock workers. */ - int num_local_scheduler_conns; - /** Local scheduler client connections. */ - LocalSchedulerConnection **conns; -} LocalSchedulerMock; - -/** - * Register clients of the local scheduler. This function is started in a - * separate thread so enable a blocking call to register the clients. - */ -static void register_clients(int num_mock_workers, LocalSchedulerMock *mock) { - for (int i = 0; i < num_mock_workers; ++i) { - new_client_connection(mock->loop, mock->local_scheduler_fd, - (void *) mock->local_scheduler_state, 0); - LocalSchedulerClient *worker = mock->local_scheduler_state->workers.back(); - process_message(mock->local_scheduler_state->loop, worker->sock, worker, 0); - } -} - -LocalSchedulerMock *LocalSchedulerMock_init(int num_workers, - int num_mock_workers) { - const char *node_ip_address = "127.0.0.1"; - const char *redis_addr = node_ip_address; - int redis_port = 6379; - std::unordered_map static_resource_conf; - static_resource_conf["CPU"] = INT16_MAX; - static_resource_conf["GPU"] = 0; - LocalSchedulerMock *mock = - (LocalSchedulerMock *) malloc(sizeof(LocalSchedulerMock)); - memset(mock, 0, sizeof(LocalSchedulerMock)); - mock->loop = event_loop_create(); - /* Bind to the local scheduler port and initialize the local scheduler. */ - std::string plasma_manager_socket_name = bind_ipc_sock_retry( - plasma_manager_socket_name_format, &mock->plasma_manager_fd); - mock->plasma_store_fd = - connect_ipc_sock_retry(plasma_store_socket_name, 5, 100); - std::string local_scheduler_socket_name = bind_ipc_sock_retry( - local_scheduler_socket_name_format, &mock->local_scheduler_fd); - RAY_CHECK(mock->plasma_store_fd >= 0 && mock->local_scheduler_fd >= 0); - - /* Construct worker command */ - std::stringstream worker_command_ss; - worker_command_ss << "python ../python/ray/workers/default_worker.py" - << " --node-ip-address=" << node_ip_address - << " --object-store-name=" << plasma_store_socket_name - << " --object-store-manager-name=" - << plasma_manager_socket_name - << " --local-scheduler-name=" << local_scheduler_socket_name - << " --redis-address=" << redis_addr << ":" << redis_port; - std::string worker_command = worker_command_ss.str(); - - mock->local_scheduler_state = LocalSchedulerState_init( - "127.0.0.1", mock->loop, redis_addr, redis_port, - local_scheduler_socket_name.c_str(), plasma_store_socket_name, - plasma_manager_socket_name.c_str(), NULL, false, static_resource_conf, - worker_command.c_str(), num_workers); - - /* Accept the workers as clients to the plasma manager. */ - for (int i = 0; i < num_workers; ++i) { - accept_client(mock->plasma_manager_fd); - } - - /* Connect a local scheduler client. */ - mock->num_local_scheduler_conns = num_mock_workers; - mock->conns = (LocalSchedulerConnection **) malloc( - sizeof(LocalSchedulerConnection *) * num_mock_workers); - - std::thread background_thread = - std::thread(register_clients, num_mock_workers, mock); - - for (int i = 0; i < num_mock_workers; ++i) { - mock->conns[i] = LocalSchedulerConnection_init( - local_scheduler_socket_name.c_str(), WorkerID::nil(), true, - JobID::nil(), false, Language::PYTHON); - } - - background_thread.join(); - - return mock; -} - -void LocalSchedulerMock_free(LocalSchedulerMock *mock) { - /* Disconnect clients. */ - for (int i = 0; i < mock->num_local_scheduler_conns; ++i) { - LocalSchedulerConnection_free(mock->conns[i]); - } - free(mock->conns); - - /* Kill all the workers and run the event loop again so that the task table - * updates propagate and the tasks in progress are freed. */ - while (mock->local_scheduler_state->workers.size() > 0) { - LocalSchedulerClient *worker = mock->local_scheduler_state->workers.front(); - kill_worker(mock->local_scheduler_state, worker, true, false); - } - event_loop_add_timer(mock->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(mock->loop); - - /* This also frees mock->loop. */ - LocalSchedulerState_free(mock->local_scheduler_state); - close(mock->plasma_store_fd); - close(mock->plasma_manager_fd); - free(mock); -} - -void reset_worker(LocalSchedulerMock *mock, LocalSchedulerClient *worker) { - if (worker->task_in_progress) { - Task_free(worker->task_in_progress); - worker->task_in_progress = NULL; - } -} - -/** - * Test that object reconstruction gets called. If a task gets submitted, - * assigned to a worker, and then reconstruction is triggered for its return - * value, the task should get assigned to a worker again. - */ -TEST object_reconstruction_test(void) { - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1); - LocalSchedulerConnection *worker = local_scheduler->conns[0]; - - /* Create a task with zero dependencies and one return value. */ - TaskExecutionSpec execution_spec = example_task_execution_spec(0, 1); - TaskSpec *spec = execution_spec.Spec(); - int64_t task_size = execution_spec.SpecSize(); - ObjectID return_id = TaskSpec_return(spec, 0); - - /* Add an empty object table entry for the object we want to reconstruct, to - * simulate it having been created and evicted. */ - const char *client_id = "clientid"; - /* Lookup the shard locations for the object table. */ - std::vector db_shards_addresses; - std::vector db_shards_ports; - redisContext *context = redisConnect("127.0.0.1", 6379); - get_redis_shards(context, db_shards_addresses, db_shards_ports); - redisFree(context); - /* There should only be one shard, so we can safely add the empty object - * table entry to the first one. */ - ASSERT(db_shards_addresses.size() == 1); - context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]); - redisReply *reply = (redisReply *) redisCommand( - context, "RAY.OBJECT_TABLE_ADD %b %ld %b %s", return_id.data(), - sizeof(return_id), 1, NIL_DIGEST, (size_t) DIGEST_SIZE, client_id); - freeReplyObject(reply); - reply = (redisReply *) redisCommand(context, "RAY.OBJECT_TABLE_REMOVE %b %s", - return_id.data(), sizeof(return_id), - client_id); - freeReplyObject(reply); - redisFree(context); - - pid_t pid = fork(); - if (pid == 0) { - /* Make sure we receive the task twice. First from the initial submission, - * and second from the reconstruct request. */ - int64_t task_assigned_size; - local_scheduler_submit(worker, execution_spec); - TaskSpec *task_assigned = - local_scheduler_get_task(worker, &task_assigned_size); - ASSERT_EQ(memcmp(task_assigned, spec, task_size), 0); - ASSERT_EQ(task_assigned_size, task_size); - int64_t reconstruct_task_size; - TaskSpec *reconstruct_task = - local_scheduler_get_task(worker, &reconstruct_task_size); - ASSERT_EQ(memcmp(reconstruct_task, spec, task_size), 0); - ASSERT_EQ(reconstruct_task_size, task_size); - /* Clean up. */ - free(reconstruct_task); - free(task_assigned); - LocalSchedulerMock_free(local_scheduler); - exit(0); - } else { - /* Run the event loop. NOTE: OSX appears to require the parent process to - * listen for events on the open file descriptors. */ - event_loop_add_timer(local_scheduler->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Set the task's status to TaskStatus::DONE to prevent the race condition - * that would suppress object reconstruction. */ - Task *task = Task_alloc( - execution_spec, TaskStatus::DONE, - get_db_client_id(local_scheduler->local_scheduler_state->db)); - task_table_add_task(local_scheduler->local_scheduler_state->db, task, NULL, - NULL, NULL); - - /* Trigger reconstruction, and run the event loop again. */ - ObjectID return_id = TaskSpec_return(spec, 0); - local_scheduler_reconstruct_objects(worker, - std::vector({return_id})); - event_loop_add_timer(local_scheduler->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Wait for the child process to exit and check that there are no tasks - * left in the local scheduler's task queue. Then, clean up. */ - wait(NULL); - ASSERT_EQ(num_waiting_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - ASSERT_EQ(num_dispatch_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - LocalSchedulerMock_free(local_scheduler); - PASS(); - } -} - -/** - * Test that object reconstruction gets recursively called. In a chain of - * tasks, if all inputs are lost, then reconstruction of the final object - * should trigger reconstruction of all previous tasks in the lineage. - */ -TEST object_reconstruction_recursive_test(void) { - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1); - LocalSchedulerConnection *worker = local_scheduler->conns[0]; - /* Create a chain of tasks, each one dependent on the one before it. Mark - * each object as available so that tasks will run immediately. */ - const int NUM_TASKS = 10; - std::vector specs; - specs.push_back(example_task_execution_spec(0, 1)); - for (int i = 1; i < NUM_TASKS; ++i) { - ObjectID arg_id = TaskSpec_return(specs[i - 1].Spec(), 0); - specs.push_back(example_task_execution_spec_with_args(1, 1, &arg_id)); - } - /* Lookup the shard locations for the object table. */ - const char *client_id = "clientid"; - std::vector db_shards_addresses; - std::vector db_shards_ports; - redisContext *context = redisConnect("127.0.0.1", 6379); - get_redis_shards(context, db_shards_addresses, db_shards_ports); - redisFree(context); - /* There should only be one shard, so we can safely add the empty object - * table entry to the first one. */ - ASSERT(db_shards_addresses.size() == 1); - context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]); - for (int i = 0; i < NUM_TASKS; ++i) { - ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0); - redisReply *reply = (redisReply *) redisCommand( - context, "RAY.OBJECT_TABLE_ADD %b %ld %b %s", return_id.data(), - sizeof(return_id), 1, NIL_DIGEST, (size_t) DIGEST_SIZE, client_id); - freeReplyObject(reply); - reply = (redisReply *) redisCommand( - context, "RAY.OBJECT_TABLE_REMOVE %b %s", return_id.data(), - sizeof(return_id), client_id); - freeReplyObject(reply); - } - redisFree(context); - - pid_t pid = fork(); - if (pid == 0) { - /* Submit the tasks, and make sure each one gets assigned to a worker. */ - for (int i = 0; i < NUM_TASKS; ++i) { - local_scheduler_submit(worker, specs[i]); - } - /* Make sure we receive each task from the initial submission. */ - for (int i = 0; i < NUM_TASKS; ++i) { - int64_t task_size; - TaskSpec *task_assigned = local_scheduler_get_task(worker, &task_size); - ASSERT_EQ(memcmp(task_assigned, specs[i].Spec(), specs[i].SpecSize()), 0); - ASSERT_EQ(task_size, specs[i].SpecSize()); - free(task_assigned); - } - /* Check that the workers receive all tasks in the final return object's - * lineage during reconstruction. */ - for (int i = 0; i < NUM_TASKS; ++i) { - int64_t task_assigned_size; - TaskSpec *task_assigned = - local_scheduler_get_task(worker, &task_assigned_size); - for (auto it = specs.begin(); it != specs.end(); it++) { - if (memcmp(task_assigned, it->Spec(), task_assigned_size) == 0) { - specs.erase(it); - break; - } - } - free(task_assigned); - } - ASSERT(specs.size() == 0); - LocalSchedulerMock_free(local_scheduler); - exit(0); - } else { - /* Simulate each task putting its return values in the object store so that - * the next task can run. */ - for (int i = 0; i < NUM_TASKS; ++i) { - ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0); - handle_object_available( - local_scheduler->local_scheduler_state, - local_scheduler->local_scheduler_state->algorithm_state, return_id); - } - /* Run the event loop. All tasks should now be dispatched. NOTE: OSX - * appears to require the parent process to listen for events on the open - * file descriptors. */ - event_loop_add_timer(local_scheduler->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Set the final task's status to TaskStatus::DONE to prevent the race - * condition that would suppress object reconstruction. */ - Task *last_task = Task_alloc( - specs[NUM_TASKS - 1], TaskStatus::DONE, - get_db_client_id(local_scheduler->local_scheduler_state->db)); - task_table_add_task(local_scheduler->local_scheduler_state->db, last_task, - NULL, NULL, NULL); - /* Simulate eviction of the objects, so that reconstruction is required. */ - for (int i = 0; i < NUM_TASKS; ++i) { - ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0); - handle_object_removed(local_scheduler->local_scheduler_state, return_id); - } - /* Trigger reconstruction for the last object. */ - ObjectID return_id = TaskSpec_return(specs[NUM_TASKS - 1].Spec(), 0); - local_scheduler_reconstruct_objects(worker, - std::vector({return_id})); - /* Run the event loop again. All tasks should be resubmitted. */ - event_loop_add_timer(local_scheduler->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Simulate each task putting its return values in the object store so that - * the next task can run. */ - for (int i = 0; i < NUM_TASKS; ++i) { - ObjectID return_id = TaskSpec_return(specs[i].Spec(), 0); - handle_object_available( - local_scheduler->local_scheduler_state, - local_scheduler->local_scheduler_state->algorithm_state, return_id); - } - /* Run the event loop again. All tasks should be dispatched again. */ - event_loop_add_timer(local_scheduler->loop, 500, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Wait for the child process to exit and check that there are no tasks - * left in the local scheduler's task queue. Then, clean up. */ - wait(NULL); - ASSERT_EQ(num_waiting_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - ASSERT_EQ(num_dispatch_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - specs.clear(); - LocalSchedulerMock_free(local_scheduler); - PASS(); - } -} - -/** - * Test that object reconstruction gets suppressed when there is a location - * listed for the object in the object table. - */ -TaskExecutionSpec *object_reconstruction_suppression_spec; - -void object_reconstruction_suppression_callback(ObjectID object_id, - bool success, - void *user_context) { - RAY_CHECK(success); - /* Submit the task after adding the object to the object table. */ - LocalSchedulerConnection *worker = (LocalSchedulerConnection *) user_context; - local_scheduler_submit(worker, *object_reconstruction_suppression_spec); -} - -TEST object_reconstruction_suppression_test(void) { - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1); - LocalSchedulerConnection *worker = local_scheduler->conns[0]; - - TaskExecutionSpec execution_spec = example_task_execution_spec(0, 1); - object_reconstruction_suppression_spec = &execution_spec; - ObjectID return_id = - TaskSpec_return(object_reconstruction_suppression_spec->Spec(), 0); - pid_t pid = fork(); - if (pid == 0) { - /* Make sure we receive the task once. This will block until the - * object_table_add callback completes. */ - int64_t task_assigned_size; - TaskSpec *task_assigned = - local_scheduler_get_task(worker, &task_assigned_size); - ASSERT_EQ( - memcmp(task_assigned, object_reconstruction_suppression_spec->Spec(), - object_reconstruction_suppression_spec->SpecSize()), - 0); - /* Trigger a reconstruction. We will check that no tasks get queued as a - * result of this line in the event loop process. */ - local_scheduler_reconstruct_objects(worker, - std::vector({return_id})); - /* Clean up. */ - free(task_assigned); - LocalSchedulerMock_free(local_scheduler); - exit(0); - } else { - /* Connect a plasma manager client so we can call object_table_add. */ - std::vector db_connect_args; - db_connect_args.push_back("manager_address"); - db_connect_args.push_back("127.0.0.1:12346"); - DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager", - "127.0.0.1", db_connect_args); - db_attach(db, local_scheduler->loop, false); - /* Add the object to the object table. */ - object_table_add(db, return_id, 1, (unsigned char *) NIL_DIGEST, NULL, - object_reconstruction_suppression_callback, - (void *) worker); - /* Run the event loop. NOTE: OSX appears to require the parent process to - * listen for events on the open file descriptors. */ - event_loop_add_timer(local_scheduler->loop, 1000, - (event_loop_timer_handler) timeout_handler, NULL); - event_loop_run(local_scheduler->loop); - /* Wait for the child process to exit and check that there are no tasks - * left in the local scheduler's task queue. Then, clean up. */ - wait(NULL); - ASSERT_EQ(num_waiting_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - ASSERT_EQ(num_dispatch_tasks( - local_scheduler->local_scheduler_state->algorithm_state), - 0); - db_disconnect(db); - LocalSchedulerMock_free(local_scheduler); - PASS(); - } -} - -TEST task_dependency_test(void) { - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1); - LocalSchedulerState *state = local_scheduler->local_scheduler_state; - SchedulingAlgorithmState *algorithm_state = state->algorithm_state; - /* Get the first worker. */ - LocalSchedulerClient *worker = state->workers.front(); - TaskExecutionSpec execution_spec = example_task_execution_spec(1, 1); - TaskSpec *spec = execution_spec.Spec(); - ObjectID oid = TaskSpec_arg_id(spec, 0, 0); - - /* Check that the task gets queued in the waiting queue if the task is - * submitted, but the input and workers are not available. */ - handle_task_submitted(state, algorithm_state, execution_spec); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Once the input is available, the task gets moved to the dispatch queue. */ - handle_object_available(state, algorithm_state, oid); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* Once a worker is available, the task gets assigned. */ - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - reset_worker(local_scheduler, worker); - - /* Check that the task gets queued in the waiting queue if the task is - * submitted and a worker is available, but the input is not. */ - handle_object_removed(state, oid); - handle_task_submitted(state, algorithm_state, execution_spec); - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Once the input is available, the task gets assigned. */ - handle_object_available(state, algorithm_state, oid); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - reset_worker(local_scheduler, worker); - - /* Check that the task gets queued in the dispatch queue if the task is - * submitted and the input is available, but no worker is available yet. */ - handle_task_submitted(state, algorithm_state, execution_spec); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* Once a worker is available, the task gets assigned. */ - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - reset_worker(local_scheduler, worker); - - /* If an object gets removed, check the first scenario again, where the task - * gets queued in the waiting task if the task is submitted and a worker is - * available, but the input is not. */ - handle_task_submitted(state, algorithm_state, execution_spec); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* If the input is removed while a task is in the dispatch queue, the task - * gets moved back to the waiting queue. */ - handle_object_removed(state, oid); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Once the input is available, the task gets moved back to the dispatch - * queue. */ - handle_object_available(state, algorithm_state, oid); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* Once a worker is available, the task gets assigned. */ - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - - LocalSchedulerMock_free(local_scheduler); - PASS(); -} - -TEST task_multi_dependency_test(void) { - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(0, 1); - LocalSchedulerState *state = local_scheduler->local_scheduler_state; - SchedulingAlgorithmState *algorithm_state = state->algorithm_state; - /* Get the first worker. */ - LocalSchedulerClient *worker = state->workers.front(); - TaskExecutionSpec execution_spec = example_task_execution_spec(2, 1); - TaskSpec *spec = execution_spec.Spec(); - ObjectID oid1 = TaskSpec_arg_id(spec, 0, 0); - ObjectID oid2 = TaskSpec_arg_id(spec, 1, 0); - - /* Check that the task gets queued in the waiting queue if the task is - * submitted, but the inputs and workers are not available. */ - handle_task_submitted(state, algorithm_state, execution_spec); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Check that the task stays in the waiting queue if only one input becomes - * available. */ - handle_object_available(state, algorithm_state, oid2); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Once all inputs are available, the task is moved to the dispatch queue. */ - handle_object_available(state, algorithm_state, oid1); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* Once a worker is available, the task gets assigned. */ - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - reset_worker(local_scheduler, worker); - - /* Check that the task gets queued in the dispatch queue if the task is - * submitted and the inputs are available, but no worker is available yet. */ - handle_task_submitted(state, algorithm_state, execution_spec); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* If any input is removed while a task is in the dispatch queue, the task - * gets moved back to the waiting queue. */ - handle_object_removed(state, oid1); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - handle_object_removed(state, oid2); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Check that the task stays in the waiting queue if only one input becomes - * available. */ - handle_object_available(state, algorithm_state, oid2); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Check that the task stays in the waiting queue if the one input is - * unavailable again. */ - handle_object_removed(state, oid2); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Check that the task stays in the waiting queue if the other input becomes - * available. */ - handle_object_available(state, algorithm_state, oid1); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 1); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - /* Once all inputs are available, the task is moved to the dispatch queue. */ - handle_object_available(state, algorithm_state, oid2); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 1); - /* Once a worker is available, the task gets assigned. */ - handle_worker_available(state, algorithm_state, worker); - ASSERT_EQ(num_waiting_tasks(algorithm_state), 0); - ASSERT_EQ(num_dispatch_tasks(algorithm_state), 0); - reset_worker(local_scheduler, worker); - - LocalSchedulerMock_free(local_scheduler); - PASS(); -} - -TEST start_kill_workers_test(void) { - /* Start some workers. */ - int num_workers = 4; - LocalSchedulerMock *local_scheduler = LocalSchedulerMock_init(num_workers, 0); - /* We start off with num_workers children processes, but no workers - * registered yet. */ - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), - static_cast(num_workers)); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), 0); - - /* Make sure that each worker connects to the local_scheduler scheduler. This - * for loop will hang if one of the workers does not connect. */ - for (int i = 0; i < num_workers; ++i) { - new_client_connection(local_scheduler->loop, - local_scheduler->local_scheduler_fd, - (void *) local_scheduler->local_scheduler_state, 0); - } - - /* After handling each worker's initial connection, we should now have all - * workers accounted for, but we haven't yet matched up process IDs with our - * children processes. */ - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), - static_cast(num_workers)); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers)); - - /* Each worker should register its process ID. */ - for (auto const &worker : local_scheduler->local_scheduler_state->workers) { - process_message(local_scheduler->local_scheduler_state->loop, worker->sock, - worker, 0); - } - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers)); - - /* After killing a worker, its state is cleaned up. */ - LocalSchedulerClient *worker = - local_scheduler->local_scheduler_state->workers.front(); - kill_worker(local_scheduler->local_scheduler_state, worker, false, false); - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers - 1)); - - /* Start a worker after the local scheduler has been initialized. */ - start_worker(local_scheduler->local_scheduler_state); - /* Accept the workers as clients to the plasma manager. */ - int new_worker_fd = accept_client(local_scheduler->plasma_manager_fd); - /* The new worker should register its process ID. */ - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 1); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers - 1)); - /* Make sure the new worker connects to the local_scheduler scheduler. */ - new_client_connection(local_scheduler->loop, - local_scheduler->local_scheduler_fd, - (void *) local_scheduler->local_scheduler_state, 0); - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 1); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers)); - /* Make sure that the new worker registers its process ID. */ - worker = local_scheduler->local_scheduler_state->workers.back(); - process_message(local_scheduler->local_scheduler_state->loop, worker->sock, - worker, 0); - ASSERT_EQ(local_scheduler->local_scheduler_state->child_pids.size(), 0); - ASSERT_EQ(local_scheduler->local_scheduler_state->workers.size(), - static_cast(num_workers)); - - /* Clean up. */ - close(new_worker_fd); - LocalSchedulerMock_free(local_scheduler); - PASS(); -} - -SUITE(local_scheduler_tests) { - RUN_REDIS_TEST(object_reconstruction_test); - RUN_REDIS_TEST(object_reconstruction_recursive_test); - RUN_REDIS_TEST(object_reconstruction_suppression_test); - RUN_REDIS_TEST(task_dependency_test); - RUN_REDIS_TEST(task_multi_dependency_test); - RUN_REDIS_TEST(start_kill_workers_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - g_task_builder = make_task_builder(); - GREATEST_MAIN_BEGIN(); - RUN_SUITE(local_scheduler_tests); - GREATEST_MAIN_END(); -} diff --git a/src/local_scheduler/test/run_tests.sh b/src/local_scheduler/test/run_tests.sh deleted file mode 100644 index 9c1d7be79..000000000 --- a/src/local_scheduler/test/run_tests.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# This needs to be run in the build tree, which is normally ray/build - -# Cause the script to exit if a single command fails. -set -e - -LaunchRedis() { - port=$1 - if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then - ./src/credis/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/credis/build/src/libmember.so \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - else - ./src/common/thirdparty/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - fi -} - - -# Start the Redis shards. -LaunchRedis 6379 -LaunchRedis 6380 -sleep 1s -# Register the shard location with the primary shard. -./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 -./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 - -./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 & -sleep 0.5s -./src/local_scheduler/local_scheduler_tests -./src/common/thirdparty/redis/src/redis-cli shutdown -./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown -killall plasma_store_server diff --git a/src/local_scheduler/test/run_valgrind.sh b/src/local_scheduler/test/run_valgrind.sh deleted file mode 100644 index 6ff1dbe33..000000000 --- a/src/local_scheduler/test/run_valgrind.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# This needs to be run in the build tree, which is normally ray/build - -set -x - -# Cause the script to exit if a single command fails. -set -e - -LaunchRedis() { - port=$1 - if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then - ./src/credis/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/credis/build/src/libmember.so \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - else - ./src/common/thirdparty/redis/src/redis-server \ - --loglevel warning \ - --loadmodule ./src/common/redis_module/libray_redis_module.so \ - --port $port & - fi -} - - -# Start the Redis shards. -LaunchRedis 6379 -LaunchRedis 6380 -sleep 1s - -# Register the shard location with the primary shard. -./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 -./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 - -./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 & -sleep 0.5s -valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests -./src/common/thirdparty/redis/src/redis-cli shutdown -./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown -killall plasma_store_server diff --git a/src/plasma/CMakeLists.txt b/src/plasma/CMakeLists.txt deleted file mode 100644 index 5037a54da..000000000 --- a/src/plasma/CMakeLists.txt +++ /dev/null @@ -1,61 +0,0 @@ -cmake_minimum_required(VERSION 3.4) - -project(plasma) - -include_directories(${CMAKE_CURRENT_LIST_DIR}) -include_directories(${CMAKE_CURRENT_LIST_DIR}/thirdparty) - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -O3") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -O3 -Werror -Wall") - -if(UNIX AND NOT APPLE) - link_libraries(rt) -endif() - -include_directories("${ARROW_INCLUDE_DIR}") - -set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs") -set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/) - -set(PLASMA_FBS_OUTPUT_FILES - "${OUTPUT_DIR}/plasma_generated.h" - "${OUTPUT_DIR}/common_generated.h") - -add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES}) -add_dependencies(gen_plasma_fbs arrow_ep) - -# Copy the fbs files from Arrow project to local directory. -add_custom_command( - OUTPUT ${PLASMA_FBS_SRC} - COMMAND mkdir -p ${CMAKE_CURRENT_LIST_DIR}/format/ - COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/ - COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/ - COMMENT "Copying ${PLASMA_FBS_SRC} to local" - VERBATIM) - -# Compile flatbuffers -add_custom_command( - OUTPUT ${PLASMA_FBS_OUTPUT_FILES} - # The --gen-object-api flag generates a C++ class MessageT for each - # flatbuffers message Message, which can be used to store deserialized - # messages in data structures. This is currently used for ObjectInfo for - # example. - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api --scoped-enums - DEPENDS ${PLASMA_FBS_SRC} - COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}" - VERBATIM) - -include_directories("${FLATBUFFERS_INCLUDE_DIR}") - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") - -add_executable(plasma_manager - plasma_manager.cc) -add_dependencies(plasma_manager gen_plasma_fbs) - -target_link_libraries(plasma_manager common ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY}) - -define_test(client_tests "") -define_test(manager_tests "" plasma_manager.cc) -target_link_libraries(manager_tests ${Boost_SYSTEM_LIBRARY}) -add_dependencies(manager_tests gen_plasma_fbs) diff --git a/src/plasma/doc/plasma-doxy-config b/src/plasma/doc/plasma-doxy-config deleted file mode 100644 index 9c291f838..000000000 --- a/src/plasma/doc/plasma-doxy-config +++ /dev/null @@ -1,2473 +0,0 @@ -# Doxyfile 1.8.13 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv -# for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = "Plasma" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 2 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:\n" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 0 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = ../src - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = ../src/utarray.h ../src/uthash.h - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- -# folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /