diff --git a/flask_s3.py b/flask_s3.py index b83fffe..f165330 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -4,12 +4,11 @@ import hashlib import json from collections import defaultdict +import boto3 +import boto3.exceptions +from botocore.exceptions import ClientError from flask import url_for as flask_url_for from flask import current_app -from boto.s3.connection import S3Connection -from boto.s3 import connect_to_region -from boto.exception import S3CreateError, S3ResponseError -from boto.s3.key import Key logger = logging.getLogger('flask_s3') @@ -122,7 +121,7 @@ def _static_folder_path(static_url, static_folder, static_asset): return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')) -def _write_files(app, static_url_loc, static_folder, files, bucket, +def _write_files(s3, app, static_url_loc, static_folder, files, bucket, ex_keys=None, hashes=None): """ Writes all the files inside a static folder to S3. """ new_hashes = [] @@ -130,7 +129,7 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, for file_path in files: asset_loc = _path_to_relative_url(file_path) key_name = _static_folder_path(static_url_loc, static_folder_rel, - asset_loc) + asset_loc).lstrip("/") msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) @@ -145,20 +144,20 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, if ex_keys and key_name in ex_keys or exclude: logger.debug("%s excluded from upload" % key_name) else: - k = Key(bucket=bucket, name=key_name) - # Set custom headers - for header, value in app.config['S3_HEADERS'].iteritems(): - k.set_metadata(header, value) - k.set_contents_from_filename(file_path) - k.make_public() + with open(file_path) as fp: + s3.put_object(Bucket=bucket, + Key=key_name, + Body=fp.read(), + ACL="public-read", + Metadata=app.config['S3_HEADERS']) return new_hashes -def _upload_files(app, files_, bucket, hashes=None): +def _upload_files(s3, app, files_, bucket, hashes=None): new_hashes = [] for (static_folder, static_url), names in files_.iteritems(): - new_hashes.extend(_write_files(app, static_url, static_folder, names, + new_hashes.extend(_write_files(s3, app, static_url, static_folder, names, bucket, hashes=hashes)) return new_hashes @@ -227,45 +226,42 @@ def create_all(app, user=None, password=None, bucket_name=None, logger.debug("All valid files: %s" % all_files) # connect to s3 - if not location: - conn = S3Connection(user, password) # (default region) - else: - conn = connect_to_region(location, - aws_access_key_id=user, - aws_secret_access_key=password) + s3 = boto3.client("s3", + region_name=location or None, + aws_access_key_id=user, + aws_secret_access_key=password) # get_or_create bucket try: - try: - bucket = conn.create_bucket(bucket_name) - except S3CreateError as e: - if e.error_code == u'BucketAlreadyOwnedByYou': - bucket = conn.get_bucket(bucket_name) - else: - raise e + s3.head_bucket(Bucket=bucket_name) + except ClientError as e: + if int(e.response['Error']['Code']) == 404: + # Create the bucket + bucket = s3.create_bucket(Bucket=bucket_name) + else: + raise - bucket.make_public(recursive=False) - except S3CreateError as e: - raise e + s3.put_bucket_acl(Bucket=bucket_name, ACL='public-read') if app.config['S3_ONLY_MODIFIED']: try: - hashes = json.loads( - Key(bucket=bucket, - name=".file-hashes").get_contents_as_string()) - except S3ResponseError as e: + hashes_object = s3.get_object(Bucket=bucket_name, Key='.file-hashes') + hashes = json.loads(str(hashes_object['Body'].read())) + except ClientError as e: logger.warn("No file hashes found: %s" % e) hashes = None - new_hashes = _upload_files(app, all_files, bucket, hashes=hashes) + new_hashes = _upload_files(s3, app, all_files, bucket_name, hashes=hashes) try: - k = Key(bucket=bucket, name=".file-hashes") - k.set_contents_from_string(json.dumps(dict(new_hashes))) - except S3ResponseError as e: + s3.put_object(Bucket=bucket_name, + Key='.file-hashes', + Body=json.dumps(dict(new_hashes)), + ACL='private') + except boto3.exceptions.S3UploadFailedError as e: logger.warn("Unable to upload file hashes: %s" % e) else: - _upload_files(app, all_files, bucket) + _upload_files(s3, app, all_files, bucket_name) class FlaskS3(object): @@ -279,6 +275,7 @@ class FlaskS3(object): :param app: optional :class:`flask.Flask` application object :type app: :class:`flask.Flask` or None """ + def __init__(self, app=None): if app is not None: self.init_app(app) diff --git a/setup.py b/setup.py index dfa4e22..35daad5 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup setup( name='Flask-S3', - version='0.1.7', + version='0.2.0', url='http://github.com/e-dard/flask-s3', license='WTFPL', author='Edward Robinson', @@ -22,7 +22,7 @@ setup( platforms='any', install_requires=[ 'Flask', - 'Boto>=2.5.2' + 'Boto3>=1.1.1' ], tests_require=['nose', 'mock'], classifiers=[ diff --git a/tests/test_flask_static.py b/tests/test_flask_static.py index 152e34d..2b1f072 100644 --- a/tests/test_flask_static.py +++ b/tests/test_flask_static.py @@ -1,10 +1,11 @@ import unittest import ntpath import tempfile -import os.path import os -from mock import Mock, patch, call +import pprint + +from mock import Mock, patch, call, mock_open from flask import Flask, render_template_string, Blueprint import flask_s3 @@ -15,6 +16,7 @@ class FlaskStaticTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) self.app.testing = True + @self.app.route('/') def a(url_for_string): return render_template_string(url_for_string) @@ -57,11 +59,12 @@ class UrlTests(unittest.TestCase): return render_template_string("{{url_for('b')}}") bp = Blueprint('admin', __name__, static_folder='admin-static') + @bp.route('/') def c(): return render_template_string("{{url_for('b')}}") - self.app.register_blueprint(bp) + self.app.register_blueprint(bp) def client_get(self, ufs): FlaskS3(self.app) @@ -140,7 +143,6 @@ class UrlTests(unittest.TestCase): class S3Tests(unittest.TestCase): - def setUp(self): self.app = Flask(__name__) self.app.testing = True @@ -234,7 +236,8 @@ class S3Tests(unittest.TestCase): actual = flask_s3._path_to_relative_url(in_) self.assertEquals(exp, actual) - @patch('flask_s3.Key') + @patch('flask_s3.boto3') + @patch('flask_s3.open', mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' @@ -247,11 +250,11 @@ class S3Tests(unittest.TestCase): call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename('/home/z/bar.css')] - flask_s3._write_files(self.app, static_url_loc, static_folder, assets, + flask_s3._write_files(key_mock, self.app, static_url_loc, static_folder, assets, None, exclude) self.assertLessEqual(expected, key_mock.mock_calls) - @patch('flask_s3.Key') + @patch('flask_s3.boto3') def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['S3_ONLY_MODIFIED'] = True @@ -260,23 +263,31 @@ class S3Tests(unittest.TestCase): filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']] expected = [] + def IntIterator(): + i = 0 + while True: + i += 1 + yield i + + data_iter = IntIterator() for filename in filenames: # Write random data into files with open(filename, 'wb') as f: - f.write(os.urandom(1024)) + data = str(data_iter.next()) + f.write(data) # We expect each file to be uploaded - expected.extend([call(bucket=None, name=filename), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filename), - call().make_public()]) + expected.extend([call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filename, + Body=data)]) files = {(static_url_loc, static_folder): filenames} - hashes = flask_s3._upload_files(self.app, files, None) + hashes = flask_s3._upload_files(key_mock, self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(expected, key_mock.mock_calls) @@ -284,18 +295,19 @@ class S3Tests(unittest.TestCase): # We now modify the second file with open(filenames[1], 'wb') as f: - f.write(os.urandom(1024)) + data = str(next(data_iter)) + f.write(data) # We expect only this file to be uploaded - expected.extend([call(bucket=None, name=filenames[1]), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filenames[1]), - call().make_public()]) + expected.extend([call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filenames[1], + Body=data)]) - new_hashes = flask_s3._upload_files(self.app, files, None, - hashes=dict(hashes)) + new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, + hashes=dict(hashes)) self.assertEqual(expected, key_mock.mock_calls) @@ -309,5 +321,6 @@ class S3Tests(unittest.TestCase): for i, e in zip(inputs, expected): self.assertEquals(e, flask_s3._static_folder_path(*i)) + if __name__ == '__main__': unittest.main()