Moved Imagenet loading library to example applications; Changed code to return filenames as well as arrays (#257)

.travis.yml

@@ -24,4 +24,3 @@ script:
   - cd test
   - python runtest.py
   - python array_test.py
-  - python datasets_test.py

doc/install-on-macosx.md

@@ -16,10 +16,10 @@ install [Anaconda](https://www.continuum.io/downloads).
 
 ```
 brew update
-brew install git cmake automake autoconf libtool boost libjpeg graphviz
+brew install git cmake automake autoconf libtool boost graphviz
 sudo easy_install pip
 sudo pip install ipython --user
-sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz --ignore-installed six
+sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz --ignore-installed six
 ```
 
 ### Build

doc/install-on-ubuntu.md

@@ -14,8 +14,8 @@ First install the dependencies. We currently do not support Python 3.
 
 ```
 sudo apt-get update
-sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz
-sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz
+sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz
+sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz
 ```
 
 ### Build

examples/imagenet/README.md

@@ -0,0 +1,14 @@
+Dependencies for Imagenet  
+
+**On Ubuntu**
+
+```
+sudo apt-get install libjpeg8-dev
+sudo pip install boto3 botocore pillow
+``` 
+**On Mac OSX**
+
+```
+brew install libjpeg
+sudo pip install boto3 botocore pillow
+```

examples/imagenet/driver.py

@@ -3,7 +3,7 @@ import boto3
 import os
 import numpy as np
 import ray
-import ray.datasets.imagenet as imagenet
+import imagenet
 
 import functions
 

examples/imagenet/functions.py

@@ -1,18 +1,18 @@
 import numpy as np
-from typing import List
+from typing import List, Tuple
 import ray
 import ray.array.remote as ra
 
-@ray.remote([List[ray.ObjRef]], [int])
+@ray.remote([List[Tuple[ray.ObjRef, ray.ObjRef]]], [int])
 def num_images(batches):
-  shape_refs = [ra.shape(batch) for batch in batches]
+  shape_refs = [ra.shape(batch[0]) for batch in batches]
   return sum([ray.get(shape_ref)[0] for shape_ref in shape_refs])
 
-@ray.remote([List[ray.ObjRef]], [np.ndarray])
+@ray.remote([List[Tuple[ray.ObjRef, ray.ObjRef]]], [np.ndarray])
 def compute_mean_image(batches):
   if len(batches) == 0:
     raise Exception("No images were passed into `compute_mean_image`.")
-  sum_image_refs = [ra.sum(batch, axis=0) for batch in batches]
+  sum_image_refs = [ra.sum(batch[0], axis=0) for batch in batches]
   sum_images = [ray.get(ref) for ref in sum_image_refs]
   n_images = num_images(batches)
   return np.sum(sum_images, axis=0).astype("float64") / ray.get(n_images)

examples/imagenet/imagenet.py

@@ -1,5 +1,5 @@
 import tarfile, io
-from typing import List
+from typing import List, Tuple
 import PIL.Image
 import numpy as np
 import boto3
@@ -21,6 +21,7 @@ def load_chunk(tarfile, size=None):
   """
 
   result = []
+  filenames = []
   for member in tarfile.getmembers():
     filename = member.path
     content = tarfile.extractfile(member)
@@ -30,9 +31,10 @@ def load_chunk(tarfile, size=None):
     if size != None:
       rgbimg = rgbimg.resize(size, PIL.Image.ANTIALIAS)
     result.append(np.array(rgbimg).reshape(1, rgbimg.size[0], rgbimg.size[1], 3))
-  return np.concatenate(result)
+    filenames.append(filename)
+  return np.concatenate(result), filenames
 
-@ray.remote([str, str, List[int]], [np.ndarray])
+@ray.remote([str, str, List[int]], [np.ndarray, List])
 def load_tarfile_from_s3(bucket, s3_key, size=[]):
   """Load an imagenet .tar file.
 
@@ -55,7 +57,7 @@ def load_tarfile_from_s3(bucket, s3_key, size=[]):
   tar = tarfile.open(mode="r", fileobj=output)
   return load_chunk(tar, size=size if size != [] else None)
 
-@ray.remote([str, List[str], List[int]], [List[ray.ObjRef]])
+@ray.remote([str, List[str], List[int]], [List[Tuple[ray.ObjRef, ray.ObjRef]]])
 def load_tarfiles_from_s3(bucket, s3_keys, size=[]):
   """Load a number of imagenet .tar files.
 

examples/imagenet/worker.py

@@ -2,7 +2,7 @@ import sys
 import argparse
 import numpy as np
 
-import ray.datasets.imagenet
+import imagenet
 
 import ray
 import ray.array.remote as ra
@@ -19,7 +19,7 @@ if __name__ == "__main__":
   args = parser.parse_args()
   ray.worker.connect(args.scheduler_address, args.objstore_address, args.worker_address)
 
-  ray.register_module(ray.datasets.imagenet)
+  ray.register_module(imagenet)
   ray.register_module(functions)
   ray.register_module(ra)
   ray.register_module(ra.random)

install-dependencies.sh

@@ -30,12 +30,12 @@ fi
 if [[ $platform == "linux" ]]; then
   # These commands must be kept in sync with the installation instructions.
   sudo apt-get update
-  sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip libjpeg8-dev graphviz
-  sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 boto3 botocore Pillow colorama graphviz
+  sudo apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz
+  sudo pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz
 elif [[ $platform == "macosx" ]]; then
   # These commands must be kept in sync with the installation instructions.
-  brew install git cmake automake autoconf libtool boost libjpeg graphviz
+  brew install git cmake automake autoconf libtool boost graphviz
   sudo easy_install pip
   sudo pip install ipython --user
-  sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 boto3 botocore Pillow colorama graphviz --ignore-installed six
+  sudo pip install numpy typing funcsigs subprocess32 protobuf==3.0.0-alpha-2 colorama graphviz --ignore-installed six
 fi

lib/python/ray/worker.py

@@ -233,7 +233,7 @@ def get(objref, worker=global_worker):
     print_task_info(ray.lib.task_info(worker.handle), worker.mode)
   value = worker.get_object(objref)
   if isinstance(value, RayFailedObject):
-    raise Exception("The task that created this object reference failed with error message: {}".format(value.error_message))
+    raise Exception("The task that created this object reference failed with error message:\n{}".format(value.error_message))
   return value
 
 def put(value, worker=global_worker):

test/datasets_test.py

@@ -1,21 +0,0 @@
-import os
-import tarfile
-import unittest
-import ray
-import ray.datasets.imagenet as imagenet
-
-class ImageNetTest(unittest.TestCase):
-
-  def testImageNetLoading(self):
-    worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_worker.py")
-    ray.services.start_ray_local(num_workers=5, worker_path=worker_path)
-
-    chunk_name = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../data/mini.tar")
-    tar = tarfile.open(chunk_name, mode= "r")
-    chunk = imagenet.load_chunk(tar, size=(256, 256))
-    self.assertEqual(chunk.shape, (2, 256, 256, 3))
-
-    ray.services.cleanup()
-
-if __name__ == "__main__":
-    unittest.main()

test/test_worker.py

@@ -5,7 +5,6 @@ import numpy as np
 import test_functions
 import ray.array.remote as ra
 import ray.array.distributed as da
-import ray.datasets.imagenet
 
 import ray