update hyperparameter optimization app (#299)

examples/hyperopt/driver.py

@@ -1,37 +1,62 @@
+# Most of the tensorflow code is adapted from Tensorflow's tutorial on using CNNs to train MNIST
+# https://www.tensorflow.org/versions/r0.9/tutorials/mnist/pros/index.html#build-a-multilayer-convolutional-network
 import numpy as np
 import ray
 import os
 
-import functions
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
 
-num_workers = 3
-samples = 50
-epochs = 100
+import hyperopt
 
-worker_dir = os.path.dirname(os.path.abspath(__file__))
-worker_path = os.path.join(worker_dir, "worker.py")
-ray.services.start_ray_local(num_workers=num_workers, worker_path=worker_path)
+if __name__ == "__main__":
+  ray.services.start_ray_local(num_workers=3)
 
-best_params = None
-best_accuracy = 0
+  # The number of sets of random hyperparameters to try.
+  trials = 2
+  # The number of training passes over the dataset to use for network.
+  epochs = 10
 
-results = []
+  # Load the mnist data and turn the data into remote objects.
+  print "Downloading the MNIST dataset. This may take a minute."
+  mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
+  train_images = ray.put(mnist.train.images)
+  train_labels = ray.put(mnist.train.labels)
+  validation_images = ray.put(mnist.validation.images)
+  validation_labels = ray.put(mnist.validation.labels)
 
-for i in range(samples):
-  learning_rate = 10 ** np.random.uniform(-6, 1)
-  batch_size = np.random.randint(30, 100)
-  dropout = np.random.uniform(0, 1)
-  stddev = 10 ** np.random.uniform(-3, 1)
-  randparams = {"learning_rate": learning_rate, "batch_size": batch_size, "dropout": dropout, "stddev": stddev}
-  results.append((randparams, functions.train_cnn(randparams, epochs)))
+  # Store the best parameters, the best accuracy, and all of the results.
+  best_params = None
+  best_accuracy = 0
+  results = []
 
-for i in range(samples):
-  params, ref = results[i]
-  accuracy = ray.get(ref)
-  print "With hyperparameters {}, we achieve an accuracy of {:.4}%.".format(params, 100 * accuracy)
-  if accuracy > best_accuracy:
-    best_params = params
-    best_accuracy = accuracy
-    print "Best parameters are now {}.".format(params)
+  # Randomly generate some hyperparameters, and launch a task for each set.
+  for i in range(trials):
+    learning_rate = 10 ** np.random.uniform(-5, 5)
+    batch_size = np.random.randint(1, 100)
+    dropout = np.random.uniform(0, 1)
+    stddev = 10 ** np.random.uniform(-5, 5)
+    params = {"learning_rate": learning_rate, "batch_size": batch_size, "dropout": dropout, "stddev": stddev}
+    results.append((params, hyperopt.train_cnn_and_compute_accuracy(params, epochs, train_images, train_labels, validation_images, validation_labels)))
 
-print "Best parameters over {} samples was {}, with an accuracy of {:.4}%.".format(samples, best_params, 100 * best_accuracy)
+  # Fetch the results of the tasks and print the results.
+  for i in range(trials):
+    params, ref = results[i]
+    accuracy = ray.get(ref)
+    print """We achieve accuracy {:.3}% with
+        learning_rate: {:.2}
+        batch_size: {}
+        dropout: {:.2}
+        stddev: {:.2}
+      """.format(100 * accuracy, params["learning_rate"], params["batch_size"], params["dropout"], params["stddev"])
+    if accuracy > best_accuracy:
+      best_params = params
+      best_accuracy = accuracy
+
+  # Record the best performing set of hyperparameters.
+  print """Best accuracy over {} trials was {:.3} with
+        learning_rate: {:.2}
+        batch_size: {}
+        dropout: {:.2}
+        stddev: {:.2}
+    """.format(trials, 100 * best_accuracy, best_params["learning_rate"], best_params["batch_size"], best_params["dropout"], best_params["stddev"])