From 060e524c92a6d970383718f38a15e9e4154e5024 Mon Sep 17 00:00:00 2001 From: krfricke Date: Thu, 11 Jun 2020 20:30:25 +0200 Subject: [PATCH] [tune] Parameter columns can now be specified in tune reporters (#8802) Co-authored-by: Richard Liaw Co-authored-by: Kai Fricke --- python/ray/tune/progress_reporter.py | 100 +++++++++++++++--- .../ray/tune/tests/test_progress_reporter.py | 90 ++++++++++++++-- 2 files changed, 164 insertions(+), 26 deletions(-) diff --git a/python/ray/tune/progress_reporter.py b/python/ray/tune/progress_reporter.py index a41ae4b38..dc0b3ba72 100644 --- a/python/ray/tune/progress_reporter.py +++ b/python/ray/tune/progress_reporter.py @@ -56,6 +56,11 @@ class TuneReporterBase(ProgressReporter): include in progress table. If this is a dict, the keys should be metric names and the values should be the displayed names. If this is a list, the metric name is used directly. + parameter_columns (dict[str, str]|list[str]): Names of parameters to + include in progress table. If this is a dict, the keys should + be parameter names and the values should be the displayed names. + If this is a list, the parameter name is used directly. If empty, + defaults to all available parameters. max_progress_rows (int): Maximum number of rows to print in the progress table. The progress table describes the progress of each trial. Defaults to 20. @@ -78,10 +83,12 @@ class TuneReporterBase(ProgressReporter): def __init__(self, metric_columns=None, + parameter_columns=None, max_progress_rows=20, max_error_rows=20, max_report_frequency=5): self._metric_columns = metric_columns or self.DEFAULT_COLUMNS + self._parameter_columns = parameter_columns or [] self._max_progress_rows = max_progress_rows self._max_error_rows = max_error_rows @@ -117,6 +124,29 @@ class TuneReporterBase(ProgressReporter): "of metric columns.") self._metric_columns.append(metric) + def add_parameter_column(self, parameter, representation=None): + """Adds a parameter to the existing columns. + + Args: + parameter (str): Parameter to add. This must be a parameter + specified in the configuration. + representation (str): Representation to use in table. Defaults to + `parameter`. + """ + if parameter in self._parameter_columns: + raise ValueError("Column {} already exists.".format(parameter)) + + if isinstance(self._parameter_columns, Mapping): + representation = representation or parameter + self._parameter_columns[parameter] = representation + else: + if representation is not None and representation != parameter: + raise ValueError( + "`representation` cannot differ from `parameter` " + "if this reporter was initialized with a list " + "of metric columns.") + self._parameter_columns.append(parameter) + def _progress_str(self, trials, done, *sys_info, fmt="psql", delim="\n"): """Returns full progress string. @@ -142,6 +172,7 @@ class TuneReporterBase(ProgressReporter): trial_progress_str( trials, metric_columns=self._metric_columns, + parameter_columns=self._parameter_columns, fmt=fmt, max_rows=max_progress)) messages.append(trial_errors_str(trials, fmt=fmt, max_rows=max_error)) @@ -157,6 +188,11 @@ class JupyterNotebookReporter(TuneReporterBase): include in progress table. If this is a dict, the keys should be metric names and the values should be the displayed names. If this is a list, the metric name is used directly. + parameter_columns (dict[str, str]|list[str]): Names of parameters to + include in progress table. If this is a dict, the keys should + be parameter names and the values should be the displayed names. + If this is a list, the parameter name is used directly. If empty, + defaults to all available parameters. max_progress_rows (int): Maximum number of rows to print in the progress table. The progress table describes the progress of each trial. Defaults to 20. @@ -170,12 +206,13 @@ class JupyterNotebookReporter(TuneReporterBase): def __init__(self, overwrite, metric_columns=None, + parameter_columns=None, max_progress_rows=20, max_error_rows=20, max_report_frequency=5): - super(JupyterNotebookReporter, - self).__init__(metric_columns, max_progress_rows, max_error_rows, - max_report_frequency) + super(JupyterNotebookReporter, self).__init__( + metric_columns, parameter_columns, max_progress_rows, + max_error_rows, max_report_frequency) self._overwrite = overwrite def report(self, trials, done, *sys_info): @@ -196,6 +233,11 @@ class CLIReporter(TuneReporterBase): include in progress table. If this is a dict, the keys should be metric names and the values should be the displayed names. If this is a list, the metric name is used directly. + parameter_columns (dict[str, str]|list[str]): Names of parameters to + include in progress table. If this is a dict, the keys should + be parameter names and the values should be the displayed names. + If this is a list, the parameter name is used directly. If empty, + defaults to all available parameters. max_progress_rows (int): Maximum number of rows to print in the progress table. The progress table describes the progress of each trial. Defaults to 20. @@ -208,12 +250,14 @@ class CLIReporter(TuneReporterBase): def __init__(self, metric_columns=None, + parameter_columns=None, max_progress_rows=20, max_error_rows=20, max_report_frequency=5): - super(CLIReporter, self).__init__(metric_columns, max_progress_rows, - max_error_rows, max_report_frequency) + super(CLIReporter, self).__init__(metric_columns, parameter_columns, + max_progress_rows, max_error_rows, + max_report_frequency) def report(self, trials, done, *sys_info): print(self._progress_str(trials, done, *sys_info)) @@ -241,7 +285,11 @@ def memory_debug_str(): "(or ray[debug]) to resolve)") -def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None): +def trial_progress_str(trials, + metric_columns, + parameter_columns=None, + fmt="psql", + max_rows=None): """Returns a human readable message for printing to the console. This contains a table where each row represents a trial, its parameters @@ -253,6 +301,11 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None): If this is a dict, the keys are metric names and the values are the names to use in the message. If this is a list, the metric name is used in the message directly. + parameter_columns (dict[str, str]|list[str]): Names of parameters to + include. If this is a dict, the keys are parameter names and the + values are the names to use in the message. If this is a list, + the parameter name is used in the message directly. If this is + empty, all parameters are used in the message. fmt (str): Output format (see tablefmt in tabulate API). max_rows (int): Maximum number of rows in the trial table. Defaults to unlimited. @@ -297,22 +350,39 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None): # Pre-process trials to figure out what columns to show. if isinstance(metric_columns, Mapping): - keys = list(metric_columns.keys()) + metric_keys = list(metric_columns.keys()) else: - keys = metric_columns - keys = [ - k for k in keys if any( + metric_keys = metric_columns + metric_keys = [ + k for k in metric_keys if any( t.last_result.get(k) is not None for t in trials) ] + + if not parameter_columns: + parameter_keys = sorted( + set().union(*[t.evaluated_params for t in trials])) + elif isinstance(parameter_columns, Mapping): + parameter_keys = list(parameter_columns.keys()) + else: + parameter_keys = parameter_columns + # Build trial rows. - params = sorted(set().union(*[t.evaluated_params for t in trials])) - trial_table = [_get_trial_info(trial, params, keys) for trial in trials] + trial_table = [ + _get_trial_info(trial, parameter_keys, metric_keys) for trial in trials + ] # Format column headings if isinstance(metric_columns, Mapping): - formatted_columns = [metric_columns[k] for k in keys] + formatted_metric_columns = [metric_columns[k] for k in metric_keys] else: - formatted_columns = keys - columns = (["Trial name", "status", "loc"] + params + formatted_columns) + formatted_metric_columns = metric_keys + if isinstance(parameter_columns, Mapping): + formatted_parameter_columns = [ + parameter_columns[k] for k in parameter_keys + ] + else: + formatted_parameter_columns = parameter_keys + columns = (["Trial name", "status", "loc"] + formatted_parameter_columns + + formatted_metric_columns) # Tabulate. messages.append( tabulate(trial_table, headers=columns, tablefmt=fmt, showindex=False)) diff --git a/python/ray/tune/tests/test_progress_reporter.py b/python/ray/tune/tests/test_progress_reporter.py index e5b928938..52c55c1c3 100644 --- a/python/ray/tune/tests/test_progress_reporter.py +++ b/python/ray/tune/tests/test_progress_reporter.py @@ -12,13 +12,13 @@ from ray.tune.progress_reporter import (CLIReporter, _fair_filter_trials, EXPECTED_RESULT_1 = """Result logdir: /foo Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED) -+--------------+------------+-------+-----+-----+ -| Trial name | status | loc | a | b | -|--------------+------------+-------+-----+-----| -| 00001 | PENDING | here | 1 | 2 | -| 00002 | RUNNING | here | 2 | 4 | -| 00000 | TERMINATED | here | 0 | 0 | -+--------------+------------+-------+-----+-----+ ++--------------+------------+-------+-----+-----+------------+ +| Trial name | status | loc | a | b | metric_1 | +|--------------+------------+-------+-----+-----+------------| +| 00001 | PENDING | here | 1 | 2 | 0.5 | +| 00002 | RUNNING | here | 2 | 4 | 1 | +| 00000 | TERMINATED | here | 0 | 0 | 0 | ++--------------+------------+-------+-----+-----+------------+ ... 2 more trials not shown (2 RUNNING)""" EXPECTED_RESULT_2 = """Result logdir: /foo @@ -33,6 +33,17 @@ Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED) | 00004 | RUNNING | here | 4 | 8 | +--------------+------------+-------+-----+-----+""" +EXPECTED_RESULT_3 = """Result logdir: /foo +Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED) ++--------------+------------+-------+-----+------------+------------+ +| Trial name | status | loc | A | Metric 1 | Metric 2 | +|--------------+------------+-------+-----+------------+------------| +| 00001 | PENDING | here | 1 | 0.5 | 0.25 | +| 00002 | RUNNING | here | 2 | 1 | 0.5 | +| 00000 | TERMINATED | here | 0 | 0 | 0 | ++--------------+------------+-------+-----+------------+------------+ +... 2 more trials not shown (2 RUNNING)""" + END_TO_END_COMMAND = """ import ray from ray import tune @@ -125,6 +136,42 @@ EXPECTED_END_TO_END_END = """Number of trials: 30 (30 TERMINATED) | f_xxxxx_00029 | TERMINATED | | | | 9 | +---------------+------------+-------+-----+-----+-----+""" +EXPECTED_END_TO_END_AC = """Number of trials: 30 (30 TERMINATED) ++---------------+------------+-------+-----+-----+-----+ +| Trial name | status | loc | a | b | c | +|---------------+------------+-------+-----+-----+-----| +| f_xxxxx_00000 | TERMINATED | | 0 | | | +| f_xxxxx_00001 | TERMINATED | | 1 | | | +| f_xxxxx_00002 | TERMINATED | | 2 | | | +| f_xxxxx_00003 | TERMINATED | | 3 | | | +| f_xxxxx_00004 | TERMINATED | | 4 | | | +| f_xxxxx_00005 | TERMINATED | | 5 | | | +| f_xxxxx_00006 | TERMINATED | | 6 | | | +| f_xxxxx_00007 | TERMINATED | | 7 | | | +| f_xxxxx_00008 | TERMINATED | | 8 | | | +| f_xxxxx_00009 | TERMINATED | | 9 | | | +| f_xxxxx_00010 | TERMINATED | | | 0 | | +| f_xxxxx_00011 | TERMINATED | | | 1 | | +| f_xxxxx_00012 | TERMINATED | | | 2 | | +| f_xxxxx_00013 | TERMINATED | | | 3 | | +| f_xxxxx_00014 | TERMINATED | | | 4 | | +| f_xxxxx_00015 | TERMINATED | | | 5 | | +| f_xxxxx_00016 | TERMINATED | | | 6 | | +| f_xxxxx_00017 | TERMINATED | | | 7 | | +| f_xxxxx_00018 | TERMINATED | | | 8 | | +| f_xxxxx_00019 | TERMINATED | | | 9 | | +| f_xxxxx_00020 | TERMINATED | | | | 0 | +| f_xxxxx_00021 | TERMINATED | | | | 1 | +| f_xxxxx_00022 | TERMINATED | | | | 2 | +| f_xxxxx_00023 | TERMINATED | | | | 3 | +| f_xxxxx_00024 | TERMINATED | | | | 4 | +| f_xxxxx_00025 | TERMINATED | | | | 5 | +| f_xxxxx_00026 | TERMINATED | | | | 6 | +| f_xxxxx_00027 | TERMINATED | | | | 7 | +| f_xxxxx_00028 | TERMINATED | | | | 8 | +| f_xxxxx_00029 | TERMINATED | | | | 9 | ++---------------+------------+-------+-----+-----+-----+""" + class ProgressReporterTest(unittest.TestCase): def mock_trial(self, status, i): @@ -202,17 +249,38 @@ class ProgressReporterTest(unittest.TestCase): t.location = "here" t.config = {"a": i, "b": i * 2} t.evaluated_params = t.config - t.last_result = {"config": {"a": i, "b": i * 2}} + t.last_result = { + "config": { + "a": i, + "b": i * 2 + }, + "metric_1": i / 2, + "metric_2": i / 4 + } t.__str__ = lambda self: self.trial_id trials.append(t) - prog1 = trial_progress_str(trials, ["a", "b"], fmt="psql", max_rows=3) + # One metric, all parameters + prog1 = trial_progress_str( + trials, ["metric_1"], None, fmt="psql", max_rows=3) print(prog1) assert prog1 == EXPECTED_RESULT_1 - prog2 = trial_progress_str( - trials, ["a", "b"], fmt="psql", max_rows=None) + + # No metric, all parameters + prog2 = trial_progress_str(trials, [], None, fmt="psql", max_rows=None) print(prog2) assert prog2 == EXPECTED_RESULT_2 + # Both metrics, one parameter, all with custom representation + prog3 = trial_progress_str( + trials, { + "metric_1": "Metric 1", + "metric_2": "Metric 2" + }, {"a": "A"}, + fmt="psql", + max_rows=3) + print(prog3) + assert prog3 == EXPECTED_RESULT_3 + def testEndToEndReporting(self): try: os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx"