wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-10 20:53:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Parameter columns can now be specified in tune reporters (#8802)

Browse Source 
Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
Co-authored-by: Kai Fricke <kai@anyscale.com>
This commit is contained in:
krfricke
2020-06-11 20:30:25 +02:00
committed by GitHub
parent e4a1cda0ad
commit 060e524c92
2 changed files with 164 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/progress_reporter.py
+85 -15
View File
@@ -56,6 +56,11 @@ class TuneReporterBase(ProgressReporter):
include in progress table. If this is a dict, the keys should
be metric names and the values should be the displayed names.
If this is a list, the metric name is used directly.
parameter_columns (dict[str, str]|list[str]): Names of parameters to
include in progress table. If this is a dict, the keys should
be parameter names and the values should be the displayed names.
If this is a list, the parameter name is used directly. If empty,
defaults to all available parameters.
max_progress_rows (int): Maximum number of rows to print
in the progress table. The progress table describes the
progress of each trial. Defaults to 20.
@@ -78,10 +83,12 @@ class TuneReporterBase(ProgressReporter):
def __init__(self,
metric_columns=None,
parameter_columns=None,
max_progress_rows=20,
max_error_rows=20,
max_report_frequency=5):
self._metric_columns = metric_columns or self.DEFAULT_COLUMNS
self._parameter_columns = parameter_columns or []
self._max_progress_rows = max_progress_rows
self._max_error_rows = max_error_rows
@@ -117,6 +124,29 @@ class TuneReporterBase(ProgressReporter):
"of metric columns.")
self._metric_columns.append(metric)
def add_parameter_column(self, parameter, representation=None):
"""Adds a parameter to the existing columns.
Args:
parameter (str): Parameter to add. This must be a parameter
specified in the configuration.
representation (str): Representation to use in table. Defaults to
`parameter`.
"""
if parameter in self._parameter_columns:
raise ValueError("Column {} already exists.".format(parameter))
if isinstance(self._parameter_columns, Mapping):
representation = representation or parameter
self._parameter_columns[parameter] = representation
else:
if representation is not None and representation != parameter:
raise ValueError(
"`representation` cannot differ from `parameter` "
"if this reporter was initialized with a list "
"of metric columns.")
self._parameter_columns.append(parameter)
def _progress_str(self, trials, done, *sys_info, fmt="psql", delim="\n"):
"""Returns full progress string.
@@ -142,6 +172,7 @@ class TuneReporterBase(ProgressReporter):
trial_progress_str(
trials,
metric_columns=self._metric_columns,
parameter_columns=self._parameter_columns,
fmt=fmt,
max_rows=max_progress))
messages.append(trial_errors_str(trials, fmt=fmt, max_rows=max_error))
@@ -157,6 +188,11 @@ class JupyterNotebookReporter(TuneReporterBase):
include in progress table. If this is a dict, the keys should
be metric names and the values should be the displayed names.
If this is a list, the metric name is used directly.
parameter_columns (dict[str, str]|list[str]): Names of parameters to
include in progress table. If this is a dict, the keys should
be parameter names and the values should be the displayed names.
If this is a list, the parameter name is used directly. If empty,
defaults to all available parameters.
max_progress_rows (int): Maximum number of rows to print
in the progress table. The progress table describes the
progress of each trial. Defaults to 20.
@@ -170,12 +206,13 @@ class JupyterNotebookReporter(TuneReporterBase):
def __init__(self,
overwrite,
metric_columns=None,
parameter_columns=None,
max_progress_rows=20,
max_error_rows=20,
max_report_frequency=5):
super(JupyterNotebookReporter,
self).__init__(metric_columns, max_progress_rows, max_error_rows,
max_report_frequency)
super(JupyterNotebookReporter, self).__init__(
metric_columns, parameter_columns, max_progress_rows,
max_error_rows, max_report_frequency)
self._overwrite = overwrite
def report(self, trials, done, *sys_info):
@@ -196,6 +233,11 @@ class CLIReporter(TuneReporterBase):
include in progress table. If this is a dict, the keys should
be metric names and the values should be the displayed names.
If this is a list, the metric name is used directly.
parameter_columns (dict[str, str]|list[str]): Names of parameters to
include in progress table. If this is a dict, the keys should
be parameter names and the values should be the displayed names.
If this is a list, the parameter name is used directly. If empty,
defaults to all available parameters.
max_progress_rows (int): Maximum number of rows to print
in the progress table. The progress table describes the
progress of each trial. Defaults to 20.
@@ -208,12 +250,14 @@ class CLIReporter(TuneReporterBase):
def __init__(self,
metric_columns=None,
parameter_columns=None,
max_progress_rows=20,
max_error_rows=20,
max_report_frequency=5):
super(CLIReporter, self).__init__(metric_columns, max_progress_rows,
max_error_rows, max_report_frequency)
super(CLIReporter, self).__init__(metric_columns, parameter_columns,
max_progress_rows, max_error_rows,
max_report_frequency)
def report(self, trials, done, *sys_info):
print(self._progress_str(trials, done, *sys_info))
@@ -241,7 +285,11 @@ def memory_debug_str():
"(or ray[debug]) to resolve)")
def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
def trial_progress_str(trials,
metric_columns,
parameter_columns=None,
fmt="psql",
max_rows=None):
"""Returns a human readable message for printing to the console.
This contains a table where each row represents a trial, its parameters
@@ -253,6 +301,11 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
If this is a dict, the keys are metric names and the values are
the names to use in the message. If this is a list, the metric
name is used in the message directly.
parameter_columns (dict[str, str]|list[str]): Names of parameters to
include. If this is a dict, the keys are parameter names and the
values are the names to use in the message. If this is a list,
the parameter name is used in the message directly. If this is
empty, all parameters are used in the message.
fmt (str): Output format (see tablefmt in tabulate API).
max_rows (int): Maximum number of rows in the trial table. Defaults to
unlimited.
@@ -297,22 +350,39 @@ def trial_progress_str(trials, metric_columns, fmt="psql", max_rows=None):
# Pre-process trials to figure out what columns to show.
if isinstance(metric_columns, Mapping):
keys = list(metric_columns.keys())
metric_keys = list(metric_columns.keys())
else:
keys = metric_columns
keys = [
k for k in keys if any(
metric_keys = metric_columns
metric_keys = [
k for k in metric_keys if any(
t.last_result.get(k) is not None for t in trials)
]
if not parameter_columns:
parameter_keys = sorted(
set().union(*[t.evaluated_params for t in trials]))
elif isinstance(parameter_columns, Mapping):
parameter_keys = list(parameter_columns.keys())
else:
parameter_keys = parameter_columns
# Build trial rows.
params = sorted(set().union(*[t.evaluated_params for t in trials]))
trial_table = [_get_trial_info(trial, params, keys) for trial in trials]
trial_table = [
_get_trial_info(trial, parameter_keys, metric_keys) for trial in trials
]
# Format column headings
if isinstance(metric_columns, Mapping):
formatted_columns = [metric_columns[k] for k in keys]
formatted_metric_columns = [metric_columns[k] for k in metric_keys]
else:
formatted_columns = keys
columns = (["Trial name", "status", "loc"] + params + formatted_columns)
formatted_metric_columns = metric_keys
if isinstance(parameter_columns, Mapping):
formatted_parameter_columns = [
parameter_columns[k] for k in parameter_keys
]
else:
formatted_parameter_columns = parameter_keys
columns = (["Trial name", "status", "loc"] + formatted_parameter_columns +
formatted_metric_columns)
# Tabulate.
messages.append(
tabulate(trial_table, headers=columns, tablefmt=fmt, showindex=False))
python/ray/tune/tests/test_progress_reporter.py
+79 -11
View File
@@ -12,13 +12,13 @@ from ray.tune.progress_reporter import (CLIReporter, _fair_filter_trials,
EXPECTED_RESULT_1 = """Result logdir: /foo
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
+--------------+------------+-------+-----+-----+
| Trial name | status | loc | a | b |
|--------------+------------+-------+-----+-----|
| 00001 | PENDING | here | 1 | 2 |
| 00002 | RUNNING | here | 2 | 4 |
| 00000 | TERMINATED | here | 0 | 0 |
+--------------+------------+-------+-----+-----+
+--------------+------------+-------+-----+-----+------------+
| Trial name | status | loc | a | b | metric_1 |
|--------------+------------+-------+-----+-----+------------|
| 00001 | PENDING | here | 1 | 2 | 0.5 |
| 00002 | RUNNING | here | 2 | 4 | 1 |
| 00000 | TERMINATED | here | 0 | 0 | 0 |
+--------------+------------+-------+-----+-----+------------+
... 2 more trials not shown (2 RUNNING)"""
EXPECTED_RESULT_2 = """Result logdir: /foo
@@ -33,6 +33,17 @@ Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
| 00004 | RUNNING | here | 4 | 8 |
+--------------+------------+-------+-----+-----+"""
EXPECTED_RESULT_3 = """Result logdir: /foo
Number of trials: 5 (1 PENDING, 3 RUNNING, 1 TERMINATED)
+--------------+------------+-------+-----+------------+------------+
| Trial name | status | loc | A | Metric 1 | Metric 2 |
|--------------+------------+-------+-----+------------+------------|
| 00001 | PENDING | here | 1 | 0.5 | 0.25 |
| 00002 | RUNNING | here | 2 | 1 | 0.5 |
| 00000 | TERMINATED | here | 0 | 0 | 0 |
+--------------+------------+-------+-----+------------+------------+
... 2 more trials not shown (2 RUNNING)"""
END_TO_END_COMMAND = """
import ray
from ray import tune
@@ -125,6 +136,42 @@ EXPECTED_END_TO_END_END = """Number of trials: 30 (30 TERMINATED)
| f_xxxxx_00029 | TERMINATED | | | | 9 |
+---------------+------------+-------+-----+-----+-----+"""
EXPECTED_END_TO_END_AC = """Number of trials: 30 (30 TERMINATED)
+---------------+------------+-------+-----+-----+-----+
| Trial name | status | loc | a | b | c |
|---------------+------------+-------+-----+-----+-----|
| f_xxxxx_00000 | TERMINATED | | 0 | | |
| f_xxxxx_00001 | TERMINATED | | 1 | | |
| f_xxxxx_00002 | TERMINATED | | 2 | | |
| f_xxxxx_00003 | TERMINATED | | 3 | | |
| f_xxxxx_00004 | TERMINATED | | 4 | | |
| f_xxxxx_00005 | TERMINATED | | 5 | | |
| f_xxxxx_00006 | TERMINATED | | 6 | | |
| f_xxxxx_00007 | TERMINATED | | 7 | | |
| f_xxxxx_00008 | TERMINATED | | 8 | | |
| f_xxxxx_00009 | TERMINATED | | 9 | | |
| f_xxxxx_00010 | TERMINATED | | | 0 | |
| f_xxxxx_00011 | TERMINATED | | | 1 | |
| f_xxxxx_00012 | TERMINATED | | | 2 | |
| f_xxxxx_00013 | TERMINATED | | | 3 | |
| f_xxxxx_00014 | TERMINATED | | | 4 | |
| f_xxxxx_00015 | TERMINATED | | | 5 | |
| f_xxxxx_00016 | TERMINATED | | | 6 | |
| f_xxxxx_00017 | TERMINATED | | | 7 | |
| f_xxxxx_00018 | TERMINATED | | | 8 | |
| f_xxxxx_00019 | TERMINATED | | | 9 | |
| f_xxxxx_00020 | TERMINATED | | | | 0 |
| f_xxxxx_00021 | TERMINATED | | | | 1 |
| f_xxxxx_00022 | TERMINATED | | | | 2 |
| f_xxxxx_00023 | TERMINATED | | | | 3 |
| f_xxxxx_00024 | TERMINATED | | | | 4 |
| f_xxxxx_00025 | TERMINATED | | | | 5 |
| f_xxxxx_00026 | TERMINATED | | | | 6 |
| f_xxxxx_00027 | TERMINATED | | | | 7 |
| f_xxxxx_00028 | TERMINATED | | | | 8 |
| f_xxxxx_00029 | TERMINATED | | | | 9 |
+---------------+------------+-------+-----+-----+-----+"""
class ProgressReporterTest(unittest.TestCase):
def mock_trial(self, status, i):
@@ -202,17 +249,38 @@ class ProgressReporterTest(unittest.TestCase):
t.location = "here"
t.config = {"a": i, "b": i * 2}
t.evaluated_params = t.config
t.last_result = {"config": {"a": i, "b": i * 2}}
t.last_result = {
"config": {
"a": i,
"b": i * 2
},
"metric_1": i / 2,
"metric_2": i / 4
}
t.__str__ = lambda self: self.trial_id
trials.append(t)
prog1 = trial_progress_str(trials, ["a", "b"], fmt="psql", max_rows=3)
# One metric, all parameters
prog1 = trial_progress_str(
trials, ["metric_1"], None, fmt="psql", max_rows=3)
print(prog1)
assert prog1 == EXPECTED_RESULT_1
prog2 = trial_progress_str(
trials, ["a", "b"], fmt="psql", max_rows=None)
# No metric, all parameters
prog2 = trial_progress_str(trials, [], None, fmt="psql", max_rows=None)
print(prog2)
assert prog2 == EXPECTED_RESULT_2
# Both metrics, one parameter, all with custom representation
prog3 = trial_progress_str(
trials, {
"metric_1": "Metric 1",
"metric_2": "Metric 2"
}, {"a": "A"},
fmt="psql",
max_rows=3)
print(prog3)
assert prog3 == EXPECTED_RESULT_3
def testEndToEndReporting(self):
try:
os.environ["_TEST_TUNE_TRIAL_UUID"] = "xxxxx"