diff --git a/python/ray/experimental/state.py b/python/ray/experimental/state.py index f0cf2f9d4..0705787f6 100644 --- a/python/ray/experimental/state.py +++ b/python/ray/experimental/state.py @@ -523,6 +523,9 @@ class GlobalState(object): del task_info[task_id_hex] heap_size -= 1 + for key, info in task_info.items(): + self._add_missing_timestamps(info) + return task_info def dump_catapult_trace(self, @@ -811,6 +814,29 @@ class GlobalState(object): all_times.append(data["store_outputs_end"]) return all_times + def _add_missing_timestamps(self, info): + """Fills in any missing timestamp values in a task info. + + Task timestamps may be missing if the task fails or is partially + executed. + """ + + keys = [ + "acquire_lock_start", + "acquire_lock_end", + "get_arguments_start", + "get_arguments_end", + "execute_start", + "execute_end", + "store_outputs_start", + "store_outputs_end"] + + latest_timestamp = 0 + for key in keys: + cur = info.get(key, latest_timestamp) + info[key] = cur + latest_timestamp = cur + def local_schedulers(self): """Get a list of live local schedulers. diff --git a/python/ray/experimental/ui.py b/python/ray/experimental/ui.py index 80e515cb1..e27797d9e 100644 --- a/python/ray/experimental/ui.py +++ b/python/ray/experimental/ui.py @@ -307,7 +307,9 @@ def _setup_trace_dependencies(): except subprocess.CalledProcessError: # Error on non-zero exit code (e.g. - ".git not found") if not os.path.exists(os.path.join(catapult_home)): - print("Cloning catapult to {}.".format(catapult_home)) + print( + "Cloning catapult to {} (this may take a while...)".format( + catapult_home)) cmd = ["git", "clone", "https://github.com/catapult-project/catapult.git",