wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-12 12:20:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update hyperparameter optimization example. (#332)

Browse Source 
* Update hyperparameter optimization example.

* Remove early stopping.
This commit is contained in:
Robert Nishihara
2017-03-04 10:45:15 -08:00
committed by Philipp Moritz
parent 41b8675d04
commit 0a233b7144
6 changed files with 453 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/hyperopt/hyperopt_adaptive.py
+141
View File
@@ -0,0 +1,141 @@
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
from collections import defaultdict
import numpy as np
import ray
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import objective
parser = argparse.ArgumentParser(description="Run the hyperparameter optimization example.")
parser.add_argument("--num-starting-segments", default=5, type=int, help="The number of training segments to start in parallel.")
parser.add_argument("--num-segments", default=10, type=int, help="The number of additional training segments to perform.")
parser.add_argument("--steps-per-segment", default=20, type=int, help="The number of steps of training to do per training segment.")
parser.add_argument("--redis-address", default=None, type=str, help="The Redis address of the cluster.")
if __name__ == "__main__":
args = parser.parse_args()
ray.init(redis_address=args.redis_address)
# The number of training passes over the dataset to use for network.
steps = args.steps_per_segment
# Load the mnist data and turn the data into remote objects.
print("Downloading the MNIST dataset. This may take a minute.")
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
train_images = ray.put(mnist.train.images)
train_labels = ray.put(mnist.train.labels)
validation_images = ray.put(mnist.validation.images)
validation_labels = ray.put(mnist.validation.labels)
# Keep track of the accuracies that we've seen at different numbers of
# iterations.
accuracies_by_num_steps = defaultdict(lambda: [])
# Define a method to determine if an experiment looks promising or not.
def is_promising(experiment_info):
accuracies = experiment_info["accuracies"]
total_num_steps = experiment_info["total_num_steps"]
comparable_accuracies = accuracies_by_num_steps[total_num_steps]
if len(comparable_accuracies) == 0:
if len(accuracies) == 1:
# This means that we haven't seen anything finish yet, so keep running
# this experiment.
return True
else:
# The experiment is promising if the second half of the accuracies are
# better than the first half of the accuracies.
return np.mean(accuracies[:len(accuracies) // 2]) < np.mean(accuracies[len(accuracies) // 2:])
# Otherwise, continue running the experiment if it is in the top half of
# experiments we've seen so far at this point in time.
return np.mean(accuracy > np.array(comparable_accuracies)) > 0.5
# Keep track of all of the experiment segments that we're running. This
# dictionary uses the object ID of the experiment as the key.
experiment_info = {}
# Keep track of the curently running experiment IDs.
remaining_ids = []
# Keep track of the best hyperparameters and the best accuracy.
best_hyperparameters = None
best_accuracy = 0
# A function for generating random hyperparameters.
def generate_hyperparameters():
return {"learning_rate": 10 ** np.random.uniform(-5, 5),
"batch_size": np.random.randint(1, 100),
"dropout": np.random.uniform(0, 1),
"stddev": 10 ** np.random.uniform(-5, 5)}
# Launch some initial experiments.
for _ in range(args.num_starting_segments):
hyperparameters = generate_hyperparameters()
experiment_id = objective.train_cnn_and_compute_accuracy.remote(
hyperparameters, steps, train_images, train_labels, validation_images,
validation_labels)
experiment_info[experiment_id] = {"hyperparameters": hyperparameters,
"total_num_steps": steps,
"accuracies": []}
remaining_ids.append(experiment_id)
for _ in range(args.num_segments):
# Wait for a segment of an experiment to finish.
ready_ids, remaining_ids = ray.wait(remaining_ids, num_returns=1)
experiment_id = ready_ids[0]
# Get the accuracy and the weights.
accuracy, weights = ray.get(experiment_id)
# Update the experiment info.
previous_info = experiment_info[experiment_id]
previous_info["accuracies"].append(accuracy)
# Update the best accuracy and best hyperparameters.
if accuracy > best_accuracy:
best_hyperparameters = hyperparameters
best_accuracy = accuracy
if is_promising(previous_info):
# If the experiment still looks promising, then continue running it.
print("Continuing to run the experiment with hyperparameters {}.".format(
previous_info["hyperparameters"]))
new_hyperparameters = previous_info["hyperparameters"]
new_info = {"hyperparameters": new_hyperparameters,
"total_num_steps": previous_info["total_num_steps"] + steps,
"accuracies": previous_info["accuracies"].copy()}
starting_weights = weights
else:
# If the experiment does not look promising, start a new experiment.
print("Ending the experiment with hyperparameters {}.".format(
previous_info["hyperparameters"]))
new_hyperparameters = generate_hyperparameters()
new_info = {"hyperparameters": new_hyperparameters,
"total_num_steps": steps,
"accuracies": []}
starting_weights = None
# Start running the next segment.
new_experiment_id = objective.train_cnn_and_compute_accuracy.remote(
new_hyperparameters, steps, train_images, train_labels,
validation_images, validation_labels, weights=starting_weights)
experiment_info[new_experiment_id] = new_info
remaining_ids.append(new_experiment_id)
# Update the set of all accuracies that we've seen.
accuracies_by_num_steps[previous_info["total_num_steps"]].append(accuracy)
# Record the best performing set of hyperparameters.
print("""Best accuracy was {:.3} with
learning_rate: {:.2}
batch_size: {}
dropout: {:.2}
stddev: {:.2}
""".format(100 * best_accuracy,
best_hyperparameters["learning_rate"],
best_hyperparameters["batch_size"],
best_hyperparameters["dropout"],
best_hyperparameters["stddev"]))
examples/hyperopt/driver.py → examples/hyperopt/hyperopt_simple.py
+31 -24
View File
@@ -1,6 +1,3 @@
# Most of the tensorflow code is adapted from Tensorflow's tutorial on using CNNs to train MNIST
# https://www.tensorflow.org/versions/r0.9/tutorials/mnist/pros/index.html#build-a-multilayer-convolutional-network
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
@@ -12,16 +9,17 @@ import argparse
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import hyperopt
import objective
parser = argparse.ArgumentParser(description="Run the hyperparameter optimization example.")
parser.add_argument("--trials", default=2, type=int, help="The number of random trials to do.")
parser.add_argument("--steps", default=10, type=int, help="The number of steps of training to do per network.")
parser.add_argument("--redis-address", default=None, type=str, help="The Redis address of the cluster.")
if __name__ == "__main__":
args = parser.parse_args()
ray.init(num_workers=10)
ray.init(redis_address=args.redis_address)
# The number of sets of random hyperparameters to try.
trials = args.trials
@@ -36,31 +34,32 @@ if __name__ == "__main__":
validation_images = ray.put(mnist.validation.images)
validation_labels = ray.put(mnist.validation.labels)
# Keep track of the best parameters and the best accuracy.
best_params = None
# Keep track of the best hyperparameters and the best accuracy.
best_hyperparamemeters = None
best_accuracy = 0
# This list holds the object IDs for all of the experiments that we have
# launched and that have not yet been processed.
remaining_ids = []
# This is a dictionary mapping the object ID of an experiment to the
# parameters used for that experiment.
params_mapping = {}
# hyerparameters used for that experiment.
hyperparameters_mapping = {}
# A function for generating random hyperparameters.
def generate_random_params():
learning_rate = 10 ** np.random.uniform(-5, 5)
batch_size = np.random.randint(1, 100)
dropout = np.random.uniform(0, 1)
stddev = 10 ** np.random.uniform(-5, 5)
return {"learning_rate": learning_rate, "batch_size": batch_size, "dropout": dropout, "stddev": stddev}
def generate_hyperparameters():
return {"learning_rate": 10 ** np.random.uniform(-5, 5),
"batch_size": np.random.randint(1, 100),
"dropout": np.random.uniform(0, 1),
"stddev": 10 ** np.random.uniform(-5, 5)}
# Randomly generate some hyperparameters, and launch a task for each set.
for i in range(trials):
params = generate_random_params()
accuracy_id = hyperopt.train_cnn_and_compute_accuracy.remote(params, steps, train_images, train_labels, validation_images, validation_labels)
hyperparameters = generate_hyperparameters()
accuracy_id = objective.train_cnn_and_compute_accuracy.remote(
hyperparameters, steps, train_images, train_labels, validation_images,
validation_labels)
remaining_ids.append(accuracy_id)
# Keep track of which parameters correspond to this experiment.
params_mapping[accuracy_id] = params
# Keep track of which hyperparameters correspond to this experiment.
hyperparameters_mapping[accuracy_id] = hyperparameters
# Fetch and print the results of the tasks in the order that they complete.
for i in range(trials):
@@ -68,16 +67,20 @@ if __name__ == "__main__":
ready_ids, remaining_ids = ray.wait(remaining_ids)
# Process the output of this task.
result_id = ready_ids[0]
params = params_mapping[result_id]
accuracy = ray.get(result_id)
hyperparameters = hyperparameters_mapping[result_id]
accuracy, _ = ray.get(result_id)
print("""We achieve accuracy {:.3}% with
learning_rate: {:.2}
batch_size: {}
dropout: {:.2}
stddev: {:.2}
""".format(100 * accuracy, params["learning_rate"], params["batch_size"], params["dropout"], params["stddev"]))
""".format(100 * accuracy,
hyperparameters["learning_rate"],
hyperparameters["batch_size"],
hyperparameters["dropout"],
hyperparameters["stddev"]))
if accuracy > best_accuracy:
best_params = params
best_hyperparameters = hyperparameters
best_accuracy = accuracy
# Record the best performing set of hyperparameters.
@@ -86,4 +89,8 @@ if __name__ == "__main__":
batch_size: {}
dropout: {:.2}
stddev: {:.2}
""".format(trials, 100 * best_accuracy, best_params["learning_rate"], best_params["batch_size"], best_params["dropout"], best_params["stddev"]))
""".format(trials, 100 * best_accuracy,
best_hyperparameters["learning_rate"],
best_hyperparameters["batch_size"],
best_hyperparameters["dropout"],
best_hyperparameters["stddev"]))
examples/hyperopt/hyperopt.py → examples/hyperopt/objective.py
+40 -30
View File
@@ -1,3 +1,7 @@
# Most of the tensorflow code is adapted from Tensorflow's tutorial on using
# CNNs to train MNIST
# https://www.tensorflow.org/versions/r0.9/tutorials/mnist/pros/index.html#build-a-multilayer-convolutional-network.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
@@ -51,42 +55,48 @@ def cnn_setup(x, y, keep_prob, lr, stddev):
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(y_conv), reduction_indices=[1]))
correct_pred = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1))
return tf.train.AdamOptimizer(lr).minimize(cross_entropy), tf.reduce_mean(tf.cast(correct_pred, tf.float32))
return tf.train.AdamOptimizer(lr).minimize(cross_entropy), tf.reduce_mean(tf.cast(correct_pred, tf.float32)), cross_entropy
# Define a remote function that takes a set of hyperparameters as well as the
# data, consructs and trains a network, and returns the validation accuracy.
@ray.remote
def train_cnn_and_compute_accuracy(params, steps, train_images, train_labels, validation_images, validation_labels):
def train_cnn_and_compute_accuracy(params, steps, train_images, train_labels,
validation_images, validation_labels,
weights=None):
# Extract the hyperparameters from the params dictionary.
learning_rate = params["learning_rate"]
batch_size = params["batch_size"]
keep = 1 - params["dropout"]
stddev = params["stddev"]
# Create the input placeholders for the network.
x = tf.placeholder(tf.float32, shape=[None, 784])
y = tf.placeholder(tf.float32, shape=[None, 10])
keep_prob = tf.placeholder(tf.float32)
# Create the network.
train_step, accuracy = cnn_setup(x, y, keep_prob, learning_rate, stddev)
# Do the training and evaluation.
with tf.Session() as sess:
# Initialize the network weights.
sess.run(tf.global_variables_initializer())
for i in range(1, steps + 1):
# Fetch the next batch of data.
image_batch = get_batch(train_images, i, batch_size)
label_batch = get_batch(train_labels, i, batch_size)
# Do one step of training.
sess.run(train_step, feed_dict={x: image_batch, y: label_batch, keep_prob: keep})
if i % 100 == 0:
# Estimate the training accuracy every once in a while.
train_ac = accuracy.eval(feed_dict={x: image_batch, y: label_batch, keep_prob: 1.0})
# If the training accuracy is too low, stop early in order to avoid
# wasting computation.
if train_ac < 0.25:
# Compute the validation accuracy and return.
totalacc = accuracy.eval(feed_dict={x: validation_images, y: validation_labels, keep_prob: 1.0})
return float(totalacc)
# Training is done, compute the validation accuracy and return.
totalacc = accuracy.eval(feed_dict={x: validation_images, y: validation_labels, keep_prob: 1.0})
return float(totalacc)
# Create the network and related variables.
with tf.Graph().as_default():
# Create the input placeholders for the network.
x = tf.placeholder(tf.float32, shape=[None, 784])
y = tf.placeholder(tf.float32, shape=[None, 10])
keep_prob = tf.placeholder(tf.float32)
# Create the network.
train_step, accuracy, loss = cnn_setup(x, y, keep_prob, learning_rate, stddev)
# Do the training and evaluation.
with tf.Session() as sess:
# Use the TensorFlowVariables utility. This is only necessary if we want to
# set and get the weights.
variables = ray.experimental.TensorFlowVariables(loss, sess)
# Initialize the network weights.
sess.run(tf.global_variables_initializer())
# If some network weights were passed in, set those.
if weights is not None:
variables.set_weights(weights)
# Do some steps of training.
for i in range(1, steps + 1):
# Fetch the next batch of data.
image_batch = get_batch(train_images, i, batch_size)
label_batch = get_batch(train_labels, i, batch_size)
# Do one step of training.
sess.run(train_step, feed_dict={x: image_batch, y: label_batch, keep_prob: keep})
# Training is done, so compute the validation accuracy and the current
# weights and return.
totalacc = accuracy.eval(feed_dict={x: validation_images,
y: validation_labels,
keep_prob: 1.0})
new_weights = variables.get_weights()
return float(totalacc), new_weights