wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-09 12:20:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Tune experiment analysis improvements (#10645)

Browse Source 
Co-authored-by: Richard Liaw <rliaw@berkeley.edu>
This commit is contained in:
Kai Fricke
2020-09-08 21:00:52 -07:00
committed by GitHub
co-authored by Richard Liaw
parent d9c68fca5c
commit d7c7aba99c
14 changed files with 247 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/tests/example.py
+1 -1
View File
@@ -39,5 +39,5 @@ print("Best config: ", analysis.get_best_config(
metric="mean_loss", mode="min"))
# Get a dataframe for analyzing trial results.
df = analysis.dataframe()
df = analysis.results_df
# __quick_start_end__
python/ray/tune/tests/test_api.py
+6 -3
View File
@@ -520,7 +520,8 @@ class TrainableFunctionApiTest(unittest.TestCase):
analysis = tune.run(train, num_samples=10, stop=stopper)
self.assertTrue(
all(t.status == Trial.TERMINATED for t in analysis.trials))
self.assertTrue(len(analysis.dataframe()) <= top)
self.assertTrue(
len(analysis.dataframe(metric="test", mode="max")) <= top)
patience = 5
stopper = EarlyStopping("test", top=top, mode="min", patience=patience)
@@ -528,14 +529,16 @@ class TrainableFunctionApiTest(unittest.TestCase):
analysis = tune.run(train, num_samples=20, stop=stopper)
self.assertTrue(
all(t.status == Trial.TERMINATED for t in analysis.trials))
self.assertTrue(len(analysis.dataframe()) <= patience)
self.assertTrue(
len(analysis.dataframe(metric="test", mode="max")) <= patience)
stopper = EarlyStopping("test", top=top, mode="min")
analysis = tune.run(train, num_samples=10, stop=stopper)
self.assertTrue(
all(t.status == Trial.TERMINATED for t in analysis.trials))
self.assertTrue(len(analysis.dataframe()) <= top)
self.assertTrue(
len(analysis.dataframe(metric="test", mode="max")) <= top)
def testBadStoppingFunction(self):
def train(config, reporter):
python/ray/tune/tests/test_experiment_analysis.py
+43 -12
View File
@@ -7,7 +7,7 @@ import pandas as pd
from numpy import nan
import ray
from ray.tune import run, sample_from
from ray import tune
from ray.tune.examples.async_hyperband_example import MyTrainableClass
@@ -26,7 +26,7 @@ class ExperimentAnalysisSuite(unittest.TestCase):
ray.shutdown()
def run_test_exp(self):
self.ea = run(
self.ea = tune.run(
MyTrainableClass,
name=self.test_name,
local_dir=self.test_dir,
@@ -34,13 +34,14 @@ class ExperimentAnalysisSuite(unittest.TestCase):
checkpoint_freq=1,
num_samples=self.num_samples,
config={
"width": sample_from(
"width": tune.sample_from(
lambda spec: 10 + int(90 * random.random())),
"height": sample_from(lambda spec: int(100 * random.random())),
"height": tune.sample_from(
lambda spec: int(100 * random.random())),
})
def nan_test_exp(self):
nan_ea = run(
nan_ea = tune.run(
lambda x: nan,
name="testing_nan",
local_dir=self.test_dir,
@@ -48,14 +49,15 @@ class ExperimentAnalysisSuite(unittest.TestCase):
checkpoint_freq=1,
num_samples=self.num_samples,
config={
"width": sample_from(
"width": tune.sample_from(
lambda spec: 10 + int(90 * random.random())),
"height": sample_from(lambda spec: int(100 * random.random())),
"height": tune.sample_from(
lambda spec: int(100 * random.random())),
})
return nan_ea
def testDataframe(self):
df = self.ea.dataframe()
df = self.ea.dataframe(self.metric, mode="max")
self.assertTrue(isinstance(df, pd.DataFrame))
self.assertEquals(df.shape[0], self.num_samples)
@@ -143,21 +145,50 @@ class ExperimentAnalysisSuite(unittest.TestCase):
self.assertEqual(df.training_iteration.max(), 1)
def testIgnoreOtherExperiment(self):
analysis = run(
analysis = tune.run(
MyTrainableClass,
name="test_example",
local_dir=self.test_dir,
stop={"training_iteration": 1},
num_samples=1,
config={
"width": sample_from(
"width": tune.sample_from(
lambda spec: 10 + int(90 * random.random())),
"height": sample_from(lambda spec: int(100 * random.random())),
"height": tune.sample_from(
lambda spec: int(100 * random.random())),
})
df = analysis.dataframe()
df = analysis.dataframe(self.metric, mode="max")
self.assertEquals(df.shape[0], 1)
class ExperimentAnalysisPropertySuite(unittest.TestCase):
def testBestProperties(self):
def train(config):
for i in range(10):
with tune.checkpoint_dir(i):
pass
tune.report(res=config["base"] + i)
ea = tune.run(
train,
config={"base": tune.grid_search([100, 200, 300])},
metric="res",
mode="max")
trials = ea.trials
self.assertEquals(ea.best_trial, trials[2])
self.assertEquals(ea.best_config, trials[2].config)
self.assertEquals(ea.best_logdir, trials[2].logdir)
self.assertEquals(ea.best_checkpoint, trials[2].checkpoint.value)
self.assertTrue(
all(ea.best_dataframe["trial_id"] == trials[2].trial_id))
self.assertEquals(ea.results_df.loc[trials[2].trial_id, "res"], 309)
self.assertEquals(ea.best_result["res"], 309)
self.assertEquals(ea.best_result_df.loc[trials[2].trial_id, "res"],
309)
if __name__ == "__main__":
import pytest
import sys
python/ray/tune/tests/test_experiment_analysis_mem.py
+5 -5
View File
@@ -83,10 +83,10 @@ class ExperimentAnalysisInMemorySuite(unittest.TestCase):
num_samples=1,
config={"id": grid_search(list(range(5)))})
max_all = ea.get_best_trial("score",
"max").metric_analysis["score"]["max"]
min_all = ea.get_best_trial("score",
"min").metric_analysis["score"]["min"]
max_all = ea.get_best_trial("score", "max",
"all").metric_analysis["score"]["max"]
min_all = ea.get_best_trial("score", "min",
"all").metric_analysis["score"]["min"]
max_last = ea.get_best_trial("score", "max",
"last").metric_analysis["score"]["last"]
max_avg = ea.get_best_trial("score", "max",
@@ -149,7 +149,7 @@ class AnalysisSuite(unittest.TestCase):
def testDataframe(self):
analysis = Analysis(self.test_dir)
df = analysis.dataframe()
df = analysis.dataframe(self.metric, mode="max")
self.assertTrue(isinstance(df, pd.DataFrame))
self.assertEqual(df.shape[0], self.num_samples * 2)
python/ray/tune/tests/test_trial_scheduler_pbt.py
+12 -3
View File
@@ -82,15 +82,24 @@ class PopulationBasedTrainingSynchTest(unittest.TestCase):
def testAsynchFail(self):
analysis = self.synchSetup(False)
self.assertTrue(any(analysis.dataframe()["mean_accuracy"] != 33))
self.assertTrue(
any(
analysis.dataframe(metric="mean_accuracy", mode="max")
["mean_accuracy"] != 33))
def testSynchPass(self):
analysis = self.synchSetup(True)
self.assertTrue(all(analysis.dataframe()["mean_accuracy"] == 33))
self.assertTrue(
all(
analysis.dataframe(metric="mean_accuracy", mode="max")[
"mean_accuracy"] == 33))
def testSynchPassLast(self):
analysis = self.synchSetup(True, param=[30, 20, 10])
self.assertTrue(all(analysis.dataframe()["mean_accuracy"] == 33))
self.assertTrue(
all(
analysis.dataframe(metric="mean_accuracy", mode="max")[
"mean_accuracy"] == 33))
class PopulationBasedTrainingConfigTest(unittest.TestCase):
python/ray/tune/tests/tutorial.py
+1 -1
View File
@@ -166,7 +166,7 @@ analysis = tune.run(train_mnist, num_samples=10, search_alg=hyperopt_search)
# __run_analysis_begin__
import os
df = analysis.dataframe()
df = analysis.results_df
logdir = analysis.get_best_logdir("mean_accuracy", mode="max")
state_dict = torch.load(os.path.join(logdir, "model.pth"))