Update to boto3 instead of boto

Reduce the total number of S3 calls by combining put_object with the
metadata and ACLs.

flask_s3.py

@@ -4,12 +4,11 @@ import hashlib
 import json
 from collections import defaultdict
 
+import boto3
+import boto3.exceptions
+from botocore.exceptions import ClientError
 from flask import url_for as flask_url_for
 from flask import current_app
-from boto.s3.connection import S3Connection
-from boto.s3 import connect_to_region
-from boto.exception import S3CreateError, S3ResponseError
-from boto.s3.key import Key
 
 logger = logging.getLogger('flask_s3')
 
@@ -122,7 +121,7 @@ def _static_folder_path(static_url, static_folder, static_asset):
     return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/'))
 
 
-def _write_files(app, static_url_loc, static_folder, files, bucket,
+def _write_files(s3, app, static_url_loc, static_folder, files, bucket,
                  ex_keys=None, hashes=None):
     """ Writes all the files inside a static folder to S3. """
     new_hashes = []
@@ -130,7 +129,7 @@ def _write_files(app, static_url_loc, static_folder, files, bucket,
     for file_path in files:
         asset_loc = _path_to_relative_url(file_path)
         key_name = _static_folder_path(static_url_loc, static_folder_rel,
-                                       asset_loc)
+                                       asset_loc).lstrip("/")
         msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name)
         logger.debug(msg)
 
@@ -145,20 +144,20 @@ def _write_files(app, static_url_loc, static_folder, files, bucket,
         if ex_keys and key_name in ex_keys or exclude:
             logger.debug("%s excluded from upload" % key_name)
         else:
-            k = Key(bucket=bucket, name=key_name)
-            # Set custom headers
-            for header, value in app.config['S3_HEADERS'].iteritems():
-                k.set_metadata(header, value)
-            k.set_contents_from_filename(file_path)
-            k.make_public()
+            with open(file_path) as fp:
+                s3.put_object(Bucket=bucket,
+                              Key=key_name,
+                              Body=fp.read(),
+                              ACL="public-read",
+                              Metadata=app.config['S3_HEADERS'])
 
     return new_hashes
 
 
-def _upload_files(app, files_, bucket, hashes=None):
+def _upload_files(s3, app, files_, bucket, hashes=None):
     new_hashes = []
     for (static_folder, static_url), names in files_.iteritems():
-        new_hashes.extend(_write_files(app, static_url, static_folder, names,
+        new_hashes.extend(_write_files(s3, app, static_url, static_folder, names,
                                        bucket, hashes=hashes))
     return new_hashes
 
@@ -227,45 +226,42 @@ def create_all(app, user=None, password=None, bucket_name=None,
     logger.debug("All valid files: %s" % all_files)
 
     # connect to s3
-    if not location:
-        conn = S3Connection(user, password)  # (default region)
-    else:
-        conn = connect_to_region(location,
-                                 aws_access_key_id=user,
-                                 aws_secret_access_key=password)
+    s3 = boto3.client("s3",
+                      region_name=location or None,
+                      aws_access_key_id=user,
+                      aws_secret_access_key=password)
 
     # get_or_create bucket
     try:
-        try:
-            bucket = conn.create_bucket(bucket_name)
-        except S3CreateError as e:
-            if e.error_code == u'BucketAlreadyOwnedByYou':
-                bucket = conn.get_bucket(bucket_name)
-            else:
-                raise e
+        s3.head_bucket(Bucket=bucket_name)
+    except ClientError as e:
+        if int(e.response['Error']['Code']) == 404:
+            # Create the bucket
+            bucket = s3.create_bucket(Bucket=bucket_name)
+        else:
+            raise
 
-        bucket.make_public(recursive=False)
-    except S3CreateError as e:
-        raise e
+    s3.put_bucket_acl(Bucket=bucket_name, ACL='public-read')
 
     if app.config['S3_ONLY_MODIFIED']:
         try:
-            hashes = json.loads(
-                Key(bucket=bucket,
-                    name=".file-hashes").get_contents_as_string())
-        except S3ResponseError as e:
+            hashes_object = s3.get_object(Bucket=bucket_name, Key='.file-hashes')
+            hashes = json.loads(str(hashes_object['Body'].read()))
+        except ClientError as e:
             logger.warn("No file hashes found: %s" % e)
             hashes = None
 
-        new_hashes = _upload_files(app, all_files, bucket, hashes=hashes)
+        new_hashes = _upload_files(s3, app, all_files, bucket_name, hashes=hashes)
 
         try:
-            k = Key(bucket=bucket, name=".file-hashes")
-            k.set_contents_from_string(json.dumps(dict(new_hashes)))
-        except S3ResponseError as e:
+            s3.put_object(Bucket=bucket_name,
+                          Key='.file-hashes',
+                          Body=json.dumps(dict(new_hashes)),
+                          ACL='private')
+        except boto3.exceptions.S3UploadFailedError as e:
             logger.warn("Unable to upload file hashes: %s" % e)
     else:
-        _upload_files(app, all_files, bucket)
+        _upload_files(s3, app, all_files, bucket_name)
 
 
 class FlaskS3(object):
@@ -279,6 +275,7 @@ class FlaskS3(object):
     :param app: optional :class:`flask.Flask` application object
     :type app: :class:`flask.Flask` or None
     """
+
     def __init__(self, app=None):
         if app is not None:
             self.init_app(app)

setup.py

@@ -9,7 +9,7 @@ from setuptools import setup
 
 setup(
     name='Flask-S3',
-    version='0.1.7',
+    version='0.2.0',
     url='http://github.com/e-dard/flask-s3',
     license='WTFPL',
     author='Edward Robinson',
@@ -22,7 +22,7 @@ setup(
     platforms='any',
     install_requires=[
         'Flask',
-        'Boto>=2.5.2'
+        'Boto3>=1.1.1'
     ],
     tests_require=['nose', 'mock'],
     classifiers=[

tests/test_flask_static.py

@@ -1,10 +1,11 @@
 import unittest
 import ntpath
 import tempfile
-import os.path
 import os
 
-from mock import Mock, patch, call
+import pprint
+
+from mock import Mock, patch, call, mock_open
 from flask import Flask, render_template_string, Blueprint
 
 import flask_s3
@@ -15,6 +16,7 @@ class FlaskStaticTest(unittest.TestCase):
     def setUp(self):
         self.app = Flask(__name__)
         self.app.testing = True
+
         @self.app.route('/<url_for_string>')
         def a(url_for_string):
             return render_template_string(url_for_string)
@@ -57,11 +59,12 @@ class UrlTests(unittest.TestCase):
             return render_template_string("{{url_for('b')}}")
 
         bp = Blueprint('admin', __name__, static_folder='admin-static')
+
         @bp.route('/<url_for_string>')
         def c():
             return render_template_string("{{url_for('b')}}")
-        self.app.register_blueprint(bp)
 
+        self.app.register_blueprint(bp)
 
     def client_get(self, ufs):
         FlaskS3(self.app)
@@ -140,7 +143,6 @@ class UrlTests(unittest.TestCase):
 
 
 class S3Tests(unittest.TestCase):
-
     def setUp(self):
         self.app = Flask(__name__)
         self.app.testing = True
@@ -234,7 +236,8 @@ class S3Tests(unittest.TestCase):
             actual = flask_s3._path_to_relative_url(in_)
             self.assertEquals(exp, actual)
 
-    @patch('flask_s3.Key')
+    @patch('flask_s3.boto3')
+    @patch('flask_s3.open', mock_open(read_data='test'))
     def test__write_files(self, key_mock):
         """ Tests _write_files """
         static_url_loc = '/foo/static'
@@ -247,11 +250,11 @@ class S3Tests(unittest.TestCase):
                     call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'),
                     call().set_metadata('Content-Encoding', 'gzip'),
                     call().set_contents_from_filename('/home/z/bar.css')]
-        flask_s3._write_files(self.app, static_url_loc, static_folder, assets,
+        flask_s3._write_files(key_mock, self.app, static_url_loc, static_folder, assets,
                               None, exclude)
         self.assertLessEqual(expected, key_mock.mock_calls)
 
-    @patch('flask_s3.Key')
+    @patch('flask_s3.boto3')
     def test__write_only_modified(self, key_mock):
         """ Test that we only upload files that have changed """
         self.app.config['S3_ONLY_MODIFIED'] = True
@@ -260,23 +263,31 @@ class S3Tests(unittest.TestCase):
         filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']]
         expected = []
 
+        def IntIterator():
+            i = 0
+            while True:
+                i += 1
+                yield i
+
+        data_iter = IntIterator()
 
         for filename in filenames:
             # Write random data into files
             with open(filename, 'wb') as f:
-                f.write(os.urandom(1024))
+                data = str(data_iter.next())
+                f.write(data)
 
             # We expect each file to be uploaded
-            expected.extend([call(bucket=None, name=filename),
-                             call().set_metadata('Expires', 
-                                 'Thu, 31 Dec 2037 23:59:59 GMT'),
-                             call().set_metadata('Content-Encoding', 'gzip'),
-                             call().set_contents_from_filename(filename),
-                             call().make_public()])
+            expected.extend([call.put_object(ACL='public-read',
+                                             Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT',
+                                                       'Content-Encoding': 'gzip'},
+                                             Bucket=None,
+                                             Key=filename,
+                                             Body=data)])
 
         files = {(static_url_loc, static_folder): filenames}
 
-        hashes = flask_s3._upload_files(self.app, files, None)
+        hashes = flask_s3._upload_files(key_mock, self.app, files, None)
 
         # All files are uploaded and hashes are returned
         self.assertLessEqual(expected, key_mock.mock_calls)
@@ -284,18 +295,19 @@ class S3Tests(unittest.TestCase):
 
         # We now modify the second file
         with open(filenames[1], 'wb') as f:
-            f.write(os.urandom(1024))
+            data = str(next(data_iter))
+            f.write(data)
 
         # We expect only this file to be uploaded
-        expected.extend([call(bucket=None, name=filenames[1]),
-                         call().set_metadata('Expires', 
-                             'Thu, 31 Dec 2037 23:59:59 GMT'),
-                         call().set_metadata('Content-Encoding', 'gzip'),
-                         call().set_contents_from_filename(filenames[1]),
-                         call().make_public()])
+        expected.extend([call.put_object(ACL='public-read',
+                                         Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT',
+                                                   'Content-Encoding': 'gzip'},
+                                         Bucket=None,
+                                         Key=filenames[1],
+                                         Body=data)])
 
-        new_hashes = flask_s3._upload_files(self.app, files, None,
-                hashes=dict(hashes))
+        new_hashes = flask_s3._upload_files(key_mock, self.app, files, None,
+                                            hashes=dict(hashes))
 
         self.assertEqual(expected, key_mock.mock_calls)
 
@@ -309,5 +321,6 @@ class S3Tests(unittest.TestCase):
         for i, e in zip(inputs, expected):
             self.assertEquals(e, flask_s3._static_folder_path(*i))
 
+
 if __name__ == '__main__':
     unittest.main()