wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-10 12:38:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Add points_to_evaluate to BasicVariantGenerator (#12916)

Browse Source 
Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
This commit is contained in:
Kai Fricke
2020-12-17 19:16:03 -08:00
committed by GitHub
co-authored by Richard Liaw
parent 124c8318a8
commit 3d72000826
11 changed files with 396 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/suggest/basic_variant.py
+98 -17
View File
@@ -1,44 +1,95 @@
import copy
import itertools
import os
import uuid
from typing import Dict, List, Union
from typing import Dict, List, Optional, Union
from ray.tune.error import TuneError
from ray.tune.experiment import Experiment, convert_to_experiment_list
from ray.tune.config_parser import make_parser, create_trial_from_spec
from ray.tune.suggest.variant_generator import (
count_variants, generate_variants, format_vars, flatten_resolved_vars)
count_variants, generate_variants, format_vars, flatten_resolved_vars,
get_preset_variants)
from ray.tune.suggest.search import SearchAlgorithm
class BasicVariantGenerator(SearchAlgorithm):
"""Uses Tune's variant generation for resolving variables.
See also: `ray.tune.suggest.variant_generator`.
This is the default search algorithm used if no other search algorithm
is specified.
User API:
Args:
points_to_evaluate (list): Initial parameter suggestions to be run
first. This is for when you already have some good parameters
you want to run first to help the algorithm make better suggestions
for future parameters. Needs to be a list of dicts containing the
configurations.
Example:
.. code-block:: python
from ray import tune
from ray.tune.suggest import BasicVariantGenerator
searcher = BasicVariantGenerator()
tune.run(my_trainable_func, algo=searcher)
# This will automatically use the `BasicVariantGenerator`
tune.run(
lambda config: config["a"] + config["b"],
config={
"a": tune.grid_search([1, 2]),
"b": tune.randint(0, 3)
},
num_samples=4)
Internal API:
In the example above, 8 trials will be generated: For each sample
(``4``), each of the grid search variants for ``a`` will be sampled
once. The ``b`` parameter will be sampled randomly.
The generator accepts a pre-set list of points that should be evaluated.
The points will replace the first samples of each experiment passed to
the ``BasicVariantGenerator``.
Each point will replace one sample of the specified ``num_samples``. If
grid search variables are overwritten with the values specified in the
presets, the number of samples will thus be reduced.
Example:
.. code-block:: python
from ray.tune.suggest import BasicVariantGenerator
from ray import tune
from ray.tune.suggest.basic_variant import BasicVariantGenerator
tune.run(
lambda config: config["a"] + config["b"],
config={
"a": tune.grid_search([1, 2]),
"b": tune.randint(0, 3)
},
search_alg=BasicVariantGenerator(points_to_evaluate=[
{"a": 2, "b": 2},
{"a": 1},
{"b": 2}
]),
num_samples=4)
The example above will produce six trials via four samples:
- The first sample will produce one trial with ``a=2`` and ``b=2``.
- The second sample will produce one trial with ``a=1`` and ``b`` sampled
randomly
- The third sample will produce two trials, one for each grid search
value of ``a``. It will be ``b=2`` for both of these trials.
- The fourth sample will produce two trials, one for each grid search
value of ``a``. ``b`` will be sampled randomly and independently for
both of these trials.
searcher = BasicVariantGenerator()
searcher.add_configurations({"experiment": { ... }})
trial = searcher.next_trial()
searcher.is_finished == True
"""
def __init__(self):
def __init__(self, points_to_evaluate: Optional[List[Dict]] = None):
"""Initializes the Variant Generator.
"""
@@ -48,6 +99,8 @@ class BasicVariantGenerator(SearchAlgorithm):
self._counter = 0
self._finished = False
self._points_to_evaluate = points_to_evaluate or []
# Unique prefix for all trials generated, e.g., trial ids start as
# 2f1e_00001, 2f1ef_00002, 2f1ef_0003, etc. Overridable for testing.
force_test_uuid = os.environ.get("_TEST_TUNE_TRIAL_UUID")
@@ -72,12 +125,14 @@ class BasicVariantGenerator(SearchAlgorithm):
"""
experiment_list = convert_to_experiment_list(experiments)
for experiment in experiment_list:
self._total_samples += count_variants(experiment.spec)
points_to_evaluate = copy.deepcopy(self._points_to_evaluate)
self._total_samples += count_variants(experiment.spec,
points_to_evaluate)
self._trial_generator = itertools.chain(
self._trial_generator,
self._generate_trials(
experiment.spec.get("num_samples", 1), experiment.spec,
experiment.dir_name))
experiment.dir_name, points_to_evaluate))
def next_trial(self):
"""Provides one Trial object to be queued into the TrialRunner.
@@ -95,7 +150,11 @@ class BasicVariantGenerator(SearchAlgorithm):
self.set_finished()
return None
def _generate_trials(self, num_samples, unresolved_spec, output_path=""):
def _generate_trials(self,
num_samples,
unresolved_spec,
output_path="",
points_to_evaluate=None):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
@@ -109,6 +168,28 @@ class BasicVariantGenerator(SearchAlgorithm):
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
points_to_evaluate = points_to_evaluate or []
while points_to_evaluate:
config = points_to_evaluate.pop(0)
for resolved_vars, spec in get_preset_variants(
unresolved_spec, config):
trial_id = self._uuid_prefix + ("%05d" % self._counter)
experiment_tag = str(self._counter)
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
num_samples -= 1
if num_samples <= 0:
return
for _ in range(num_samples):
for resolved_vars, spec in generate_variants(unresolved_spec):
trial_id = self._uuid_prefix + ("%05d" % self._counter)
python/ray/tune/suggest/variant_generator.py
+73 -8
View File
@@ -1,6 +1,7 @@
import copy
import logging
from typing import Any, Dict, Generator, List, Tuple
from collections.abc import Mapping
from typing import Any, Dict, Generator, List, Optional, Tuple
import numpy
import random
@@ -138,13 +139,38 @@ def parse_spec_vars(spec: Dict) -> Tuple[List[Tuple[Tuple, Any]], List[Tuple[
return resolved_vars, domain_vars, grid_vars
def count_variants(spec: Dict) -> int:
spec = copy.deepcopy(spec)
_, domain_vars, grid_vars = parse_spec_vars(spec)
grid_count = 1
for path, domain in grid_vars:
grid_count *= len(domain.categories)
return spec.get("num_samples", 1) * grid_count
def count_variants(spec: Dict, presets: Optional[List[Dict]] = None) -> int:
# Helper function: Deep update dictionary
def deep_update(d, u):
for k, v in u.items():
if isinstance(v, Mapping):
d[k] = deep_update(d.get(k, {}), v)
else:
d[k] = v
return d
# Count samples for a specific spec
def spec_samples(spec, num_samples=1):
_, domain_vars, grid_vars = parse_spec_vars(spec)
grid_count = 1
for path, domain in grid_vars:
grid_count *= len(domain.categories)
return num_samples * grid_count
total_samples = 0
total_num_samples = spec.get("num_samples", 1)
# For each preset, overwrite the spec and count the samples generated
# for this preset
for preset in presets:
preset_spec = copy.deepcopy(spec)
deep_update(preset_spec["config"], preset)
total_samples += spec_samples(preset_spec, 1)
total_num_samples -= 1
# Add the remaining samples
if total_num_samples > 0:
total_samples += spec_samples(spec, total_num_samples)
return total_samples
def _generate_variants(spec: Dict) -> Tuple[Dict, Dict]:
@@ -172,6 +198,45 @@ def _generate_variants(spec: Dict) -> Tuple[Dict, Dict]:
yield resolved_vars, spec
def get_preset_variants(spec: Dict, config: Dict):
"""Get variants according to a spec, initialized with a config.
Variables from the spec are overwritten by the variables in the config.
Thus, we may end up with less sampled parameters.
This function also checks if values used to overwrite search space
parameters are valid, and logs a warning if not.
"""
spec = copy.deepcopy(spec)
resolved, _, _ = parse_spec_vars(config)
for path, val in resolved:
try:
domain = _get_value(spec["config"], path)
if isinstance(domain, dict):
if "grid_search" in domain:
domain = Categorical(domain["grid_search"])
else:
# If users want to overwrite an entire subdict,
# let them do it.
domain = None
except IndexError as exc:
raise ValueError(
f"Pre-set config key `{'/'.join(path)}` does not correspond "
f"to a valid key in the search space definition. Please add "
f"this path to the `config` variable passed to `tune.run()`."
) from exc
if domain and not domain.is_valid(val):
logger.warning(
f"Pre-set value `{val}` is not within valid values of "
f"parameter `{'/'.join(path)}`: {domain.domain_str}")
assign_value(spec["config"], path, val)
return _generate_variants(spec)
def assign_value(spec: Dict, path: Tuple, value: Any):
for k in path[:-1]:
spec = spec[k]