From 991b911e1d8360c1e20e27648eb28f39d6db9d4b Mon Sep 17 00:00:00 2001 From: Andrew Tan Date: Fri, 5 Apr 2019 19:49:01 -0700 Subject: [PATCH] [tune] Add `--columns` flag for CLI (#4564) --- doc/source/tune-usage.rst | 4 ++-- python/ray/tune/commands.py | 12 +++++++--- python/ray/tune/scripts.py | 32 ++++++++++++++++++++++---- python/ray/tune/tests/test_commands.py | 13 ++++++++++- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/doc/source/tune-usage.rst b/doc/source/tune-usage.rst index 7ef0d0f74..28879f20d 100644 --- a/doc/source/tune-usage.rst +++ b/doc/source/tune-usage.rst @@ -465,7 +465,7 @@ Tune CLI (Experimental) Here are a few examples of command line calls. -- ``tune list-trials``: List tabular information about trials within an experiment. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). +- ``tune list-trials``: List tabular information about trials within an experiment. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). Add the ``--columns`` and ``--result-columns`` flags to select specific columns to display. .. code-block:: bash @@ -494,7 +494,7 @@ Here are a few examples of command line calls. Dropped columns: ['status', 'last_update_time'] Please increase your terminal size to view remaining columns. -- ``tune list-experiments``: List tabular information about experiments within a project. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). +- ``tune list-experiments``: List tabular information about experiments within a project. Empty columns will be dropped by default. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--filter`` flag to filter the output in the format ``" "``. Add the ``--output`` flag to write the trial information to a specific file (CSV or Pickle). Add the ``--columns`` flag to select specific columns to display. .. code-block:: bash diff --git a/python/ray/tune/commands.py b/python/ray/tune/commands.py index b55e9d967..e272eb52e 100644 --- a/python/ray/tune/commands.py +++ b/python/ray/tune/commands.py @@ -129,8 +129,8 @@ def list_trials(experiment_path, sort=None, output=None, filter_op=None, - info_keys=DEFAULT_EXPERIMENT_INFO_KEYS, - result_keys=DEFAULT_RESULT_KEYS): + info_keys=None, + result_keys=None): """Lists trials in the directory subtree starting at the given path. Args: @@ -151,6 +151,10 @@ def list_trials(experiment_path, checkpoint_dicts = [flatten_dict(g) for g in checkpoint_dicts] checkpoints_df = pd.DataFrame(checkpoint_dicts) + if not info_keys: + info_keys = DEFAULT_EXPERIMENT_INFO_KEYS + if not result_keys: + result_keys = DEFAULT_RESULT_KEYS result_keys = ["last_result:{}".format(k) for k in result_keys] col_keys = [ k for k in list(info_keys) + result_keys if k in checkpoints_df @@ -208,7 +212,7 @@ def list_experiments(project_path, sort=None, output=None, filter_op=None, - info_keys=DEFAULT_PROJECT_INFO_KEYS): + info_keys=None): """Lists experiments in the directory subtree. Args: @@ -263,6 +267,8 @@ def list_experiments(project_path, sys.exit(0) info_df = pd.DataFrame(experiment_data_collection) + if not info_keys: + info_keys = DEFAULT_PROJECT_INFO_KEYS col_keys = [k for k in list(info_keys) if k in info_df] if not col_keys: print("None of keys {} in experiment data!".format(info_keys)) diff --git a/python/ray/tune/scripts.py b/python/ray/tune/scripts.py index cdf9f7b2a..64a5b79c6 100644 --- a/python/ray/tune/scripts.py +++ b/python/ray/tune/scripts.py @@ -28,9 +28,26 @@ def cli(): default=None, type=str, help="Select filter in the format ' '.") -def list_trials(experiment_path, sort, output, filter_op): +@click.option( + "--columns", + default=None, + type=str, + help="Select columns to be displayed.") +@click.option( + "--result-columns", + "result_columns", + default=None, + type=str, + help="Select columns of last result to be displayed.") +def list_trials(experiment_path, sort, output, filter_op, columns, + result_columns): """Lists trials in the directory subtree starting at the given path.""" - commands.list_trials(experiment_path, sort, output, filter_op) + if columns: + columns = columns.split(',') + if result_columns: + result_columns = result_columns.split(',') + commands.list_trials(experiment_path, sort, output, filter_op, columns, + result_columns) @cli.command() @@ -50,9 +67,16 @@ def list_trials(experiment_path, sort, output, filter_op): default=None, type=str, help="Select filter in the format ' '.") -def list_experiments(project_path, sort, output, filter_op): +@click.option( + "--columns", + default=None, + type=str, + help="Select columns to be displayed.") +def list_experiments(project_path, sort, output, filter_op, columns): """Lists experiments in the directory subtree.""" - commands.list_experiments(project_path, sort, output, filter_op) + if columns: + columns = columns.split(',') + commands.list_experiments(project_path, sort, output, filter_op, columns) @cli.command() diff --git a/python/ray/tune/tests/test_commands.py b/python/ray/tune/tests/test_commands.py index e6fcc8060..bd075dba7 100644 --- a/python/ray/tune/tests/test_commands.py +++ b/python/ray/tune/tests/test_commands.py @@ -79,9 +79,18 @@ def test_ls(start_ray, tmpdir): }) with Capturing() as output: - commands.list_trials(experiment_path, info_keys=("status", )) + commands.list_trials( + experiment_path, + info_keys=("status", ), + result_keys=( + "episode_reward_mean", + "training_iteration", + )) lines = output.captured assert sum("TERMINATED" in line for line in lines) == num_samples + columns = ["status", "episode_reward_mean", "training_iteration"] + assert all(col in lines[1] for col in columns) + assert lines[1].count('|') == 4 with Capturing() as output: commands.list_trials( @@ -113,6 +122,8 @@ def test_lsx(start_ray, tmpdir): commands.list_experiments(project_path, info_keys=("total_trials", )) lines = output.captured assert sum("1" in line for line in lines) >= num_experiments + assert "total_trials" in lines[1] + assert lines[1].count('|') == 2 with Capturing() as output: commands.list_experiments(