From 34cc79dcaee001a8232126067d474a240c997e36 Mon Sep 17 00:00:00 2001 From: Franklyn Tackitt Date: Thu, 20 Aug 2015 16:21:45 -0700 Subject: [PATCH 1/6] Update to boto3 instead of boto Reduce the total number of S3 calls by combining put_object with the metadata and ACLs. --- flask_s3.py | 75 ++++++++++++++++++-------------------- setup.py | 4 +- tests/test_flask_static.py | 61 +++++++++++++++++++------------ 3 files changed, 75 insertions(+), 65 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index b83fffe..f165330 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -4,12 +4,11 @@ import hashlib import json from collections import defaultdict +import boto3 +import boto3.exceptions +from botocore.exceptions import ClientError from flask import url_for as flask_url_for from flask import current_app -from boto.s3.connection import S3Connection -from boto.s3 import connect_to_region -from boto.exception import S3CreateError, S3ResponseError -from boto.s3.key import Key logger = logging.getLogger('flask_s3') @@ -122,7 +121,7 @@ def _static_folder_path(static_url, static_folder, static_asset): return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')) -def _write_files(app, static_url_loc, static_folder, files, bucket, +def _write_files(s3, app, static_url_loc, static_folder, files, bucket, ex_keys=None, hashes=None): """ Writes all the files inside a static folder to S3. """ new_hashes = [] @@ -130,7 +129,7 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, for file_path in files: asset_loc = _path_to_relative_url(file_path) key_name = _static_folder_path(static_url_loc, static_folder_rel, - asset_loc) + asset_loc).lstrip("/") msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) @@ -145,20 +144,20 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, if ex_keys and key_name in ex_keys or exclude: logger.debug("%s excluded from upload" % key_name) else: - k = Key(bucket=bucket, name=key_name) - # Set custom headers - for header, value in app.config['S3_HEADERS'].iteritems(): - k.set_metadata(header, value) - k.set_contents_from_filename(file_path) - k.make_public() + with open(file_path) as fp: + s3.put_object(Bucket=bucket, + Key=key_name, + Body=fp.read(), + ACL="public-read", + Metadata=app.config['S3_HEADERS']) return new_hashes -def _upload_files(app, files_, bucket, hashes=None): +def _upload_files(s3, app, files_, bucket, hashes=None): new_hashes = [] for (static_folder, static_url), names in files_.iteritems(): - new_hashes.extend(_write_files(app, static_url, static_folder, names, + new_hashes.extend(_write_files(s3, app, static_url, static_folder, names, bucket, hashes=hashes)) return new_hashes @@ -227,45 +226,42 @@ def create_all(app, user=None, password=None, bucket_name=None, logger.debug("All valid files: %s" % all_files) # connect to s3 - if not location: - conn = S3Connection(user, password) # (default region) - else: - conn = connect_to_region(location, - aws_access_key_id=user, - aws_secret_access_key=password) + s3 = boto3.client("s3", + region_name=location or None, + aws_access_key_id=user, + aws_secret_access_key=password) # get_or_create bucket try: - try: - bucket = conn.create_bucket(bucket_name) - except S3CreateError as e: - if e.error_code == u'BucketAlreadyOwnedByYou': - bucket = conn.get_bucket(bucket_name) - else: - raise e + s3.head_bucket(Bucket=bucket_name) + except ClientError as e: + if int(e.response['Error']['Code']) == 404: + # Create the bucket + bucket = s3.create_bucket(Bucket=bucket_name) + else: + raise - bucket.make_public(recursive=False) - except S3CreateError as e: - raise e + s3.put_bucket_acl(Bucket=bucket_name, ACL='public-read') if app.config['S3_ONLY_MODIFIED']: try: - hashes = json.loads( - Key(bucket=bucket, - name=".file-hashes").get_contents_as_string()) - except S3ResponseError as e: + hashes_object = s3.get_object(Bucket=bucket_name, Key='.file-hashes') + hashes = json.loads(str(hashes_object['Body'].read())) + except ClientError as e: logger.warn("No file hashes found: %s" % e) hashes = None - new_hashes = _upload_files(app, all_files, bucket, hashes=hashes) + new_hashes = _upload_files(s3, app, all_files, bucket_name, hashes=hashes) try: - k = Key(bucket=bucket, name=".file-hashes") - k.set_contents_from_string(json.dumps(dict(new_hashes))) - except S3ResponseError as e: + s3.put_object(Bucket=bucket_name, + Key='.file-hashes', + Body=json.dumps(dict(new_hashes)), + ACL='private') + except boto3.exceptions.S3UploadFailedError as e: logger.warn("Unable to upload file hashes: %s" % e) else: - _upload_files(app, all_files, bucket) + _upload_files(s3, app, all_files, bucket_name) class FlaskS3(object): @@ -279,6 +275,7 @@ class FlaskS3(object): :param app: optional :class:`flask.Flask` application object :type app: :class:`flask.Flask` or None """ + def __init__(self, app=None): if app is not None: self.init_app(app) diff --git a/setup.py b/setup.py index dfa4e22..35daad5 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup setup( name='Flask-S3', - version='0.1.7', + version='0.2.0', url='http://github.com/e-dard/flask-s3', license='WTFPL', author='Edward Robinson', @@ -22,7 +22,7 @@ setup( platforms='any', install_requires=[ 'Flask', - 'Boto>=2.5.2' + 'Boto3>=1.1.1' ], tests_require=['nose', 'mock'], classifiers=[ diff --git a/tests/test_flask_static.py b/tests/test_flask_static.py index 152e34d..2b1f072 100644 --- a/tests/test_flask_static.py +++ b/tests/test_flask_static.py @@ -1,10 +1,11 @@ import unittest import ntpath import tempfile -import os.path import os -from mock import Mock, patch, call +import pprint + +from mock import Mock, patch, call, mock_open from flask import Flask, render_template_string, Blueprint import flask_s3 @@ -15,6 +16,7 @@ class FlaskStaticTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) self.app.testing = True + @self.app.route('/') def a(url_for_string): return render_template_string(url_for_string) @@ -57,11 +59,12 @@ class UrlTests(unittest.TestCase): return render_template_string("{{url_for('b')}}") bp = Blueprint('admin', __name__, static_folder='admin-static') + @bp.route('/') def c(): return render_template_string("{{url_for('b')}}") - self.app.register_blueprint(bp) + self.app.register_blueprint(bp) def client_get(self, ufs): FlaskS3(self.app) @@ -140,7 +143,6 @@ class UrlTests(unittest.TestCase): class S3Tests(unittest.TestCase): - def setUp(self): self.app = Flask(__name__) self.app.testing = True @@ -234,7 +236,8 @@ class S3Tests(unittest.TestCase): actual = flask_s3._path_to_relative_url(in_) self.assertEquals(exp, actual) - @patch('flask_s3.Key') + @patch('flask_s3.boto3') + @patch('flask_s3.open', mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' @@ -247,11 +250,11 @@ class S3Tests(unittest.TestCase): call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename('/home/z/bar.css')] - flask_s3._write_files(self.app, static_url_loc, static_folder, assets, + flask_s3._write_files(key_mock, self.app, static_url_loc, static_folder, assets, None, exclude) self.assertLessEqual(expected, key_mock.mock_calls) - @patch('flask_s3.Key') + @patch('flask_s3.boto3') def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['S3_ONLY_MODIFIED'] = True @@ -260,23 +263,31 @@ class S3Tests(unittest.TestCase): filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']] expected = [] + def IntIterator(): + i = 0 + while True: + i += 1 + yield i + + data_iter = IntIterator() for filename in filenames: # Write random data into files with open(filename, 'wb') as f: - f.write(os.urandom(1024)) + data = str(data_iter.next()) + f.write(data) # We expect each file to be uploaded - expected.extend([call(bucket=None, name=filename), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filename), - call().make_public()]) + expected.extend([call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filename, + Body=data)]) files = {(static_url_loc, static_folder): filenames} - hashes = flask_s3._upload_files(self.app, files, None) + hashes = flask_s3._upload_files(key_mock, self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(expected, key_mock.mock_calls) @@ -284,18 +295,19 @@ class S3Tests(unittest.TestCase): # We now modify the second file with open(filenames[1], 'wb') as f: - f.write(os.urandom(1024)) + data = str(next(data_iter)) + f.write(data) # We expect only this file to be uploaded - expected.extend([call(bucket=None, name=filenames[1]), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filenames[1]), - call().make_public()]) + expected.extend([call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filenames[1], + Body=data)]) - new_hashes = flask_s3._upload_files(self.app, files, None, - hashes=dict(hashes)) + new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, + hashes=dict(hashes)) self.assertEqual(expected, key_mock.mock_calls) @@ -309,5 +321,6 @@ class S3Tests(unittest.TestCase): for i, e in zip(inputs, expected): self.assertEquals(e, flask_s3._static_folder_path(*i)) + if __name__ == '__main__': unittest.main() From e39fc3e3ed1ba812fa689bebd2c93501f9a19aa5 Mon Sep 17 00:00:00 2001 From: Franklyn Tackitt Date: Thu, 20 Aug 2015 16:33:19 -0700 Subject: [PATCH 2/6] Fix compatibility with old .file-hashes files boto3 uploads /some/key to bucket//some/key, leaving part of the path with "//", or a folder without a name. To remedy this, I had lstrip'd the / from the key before upload. I didn't think about the fact that boto2 works fine with this, and the old .file-hashes file would have the keys with the prefixing forward slash. This solves that --- flask_s3.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index f165330..138a696 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -128,20 +128,21 @@ def _write_files(s3, app, static_url_loc, static_folder, files, bucket, static_folder_rel = _path_to_relative_url(static_folder) for file_path in files: asset_loc = _path_to_relative_url(file_path) - key_name = _static_folder_path(static_url_loc, static_folder_rel, - asset_loc).lstrip("/") + full_key_name = _static_folder_path(static_url_loc, static_folder_rel, + asset_loc) + key_name = full_key_name.lstrip("/") msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) exclude = False if app.config.get('S3_ONLY_MODIFIED', False): file_hash = hash_file(file_path) - new_hashes.append((key_name, file_hash)) + new_hashes.append((full_key_name, file_hash)) - if hashes and hashes.get(key_name, None) == file_hash: + if hashes and hashes.get(full_key_name, None) == file_hash: exclude = True - if ex_keys and key_name in ex_keys or exclude: + if ex_keys and full_key_name in ex_keys or exclude: logger.debug("%s excluded from upload" % key_name) else: with open(file_path) as fp: From 5281633ba5d333dcf9536e9c4194004fb0a607c5 Mon Sep 17 00:00:00 2001 From: Franklyn Tackitt Date: Fri, 21 Aug 2015 10:44:11 -0700 Subject: [PATCH 3/6] Fix requirement in docs/requirements.txt, fix test --- docs/requirements.txt | 2 +- tests/test_flask_static.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7d01ae6..216a6f4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ Flask==0.9 Jinja2==2.6 Werkzeug==0.8.3 -boto==2.29.1 +boto3==1.1.1 wsgiref==0.1.2 diff --git a/tests/test_flask_static.py b/tests/test_flask_static.py index 2b1f072..fde9ae1 100644 --- a/tests/test_flask_static.py +++ b/tests/test_flask_static.py @@ -3,8 +3,6 @@ import ntpath import tempfile import os -import pprint - from mock import Mock, patch, call, mock_open from flask import Flask, render_template_string, Blueprint @@ -237,7 +235,7 @@ class S3Tests(unittest.TestCase): self.assertEquals(exp, actual) @patch('flask_s3.boto3') - @patch('flask_s3.open', mock_open(read_data='test')) + @patch('__builtin__.open', mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' @@ -278,12 +276,12 @@ class S3Tests(unittest.TestCase): f.write(data) # We expect each file to be uploaded - expected.extend([call.put_object(ACL='public-read', - Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', - 'Content-Encoding': 'gzip'}, - Bucket=None, - Key=filename, - Body=data)]) + expected.append(call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filename.lstrip("/"), + Body=data)) files = {(static_url_loc, static_folder): filenames} @@ -299,16 +297,18 @@ class S3Tests(unittest.TestCase): f.write(data) # We expect only this file to be uploaded - expected.extend([call.put_object(ACL='public-read', - Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', - 'Content-Encoding': 'gzip'}, - Bucket=None, - Key=filenames[1], - Body=data)]) + expected.append(call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filenames[1].lstrip("/"), + Body=data)) new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, hashes=dict(hashes)) + import pprint + pprint.pprint(zip(expected, key_mock.mock_calls)) self.assertEqual(expected, key_mock.mock_calls) def test_static_folder_path(self): From d1f544adf68ac174f3c993a51a5303941caaa7ed Mon Sep 17 00:00:00 2001 From: SunDwarf Date: Fri, 28 Aug 2015 18:09:26 +0100 Subject: [PATCH 4/6] Add Python 3 support, and merge #38. --- CONTRIBUTORS | 2 + flask_s3.py | 10 +-- setup.py | 6 +- ...st_flask_static.py => test_flask_static.py | 61 ++++++++++++------- 4 files changed, 50 insertions(+), 29 deletions(-) rename tests/test_flask_static.py => test_flask_static.py (86%) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e7de847..27fcfe2 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -5,3 +5,5 @@ Contributors * Rehan Dalal (rehandalal) * Hannes Ljungberg (hannseman) * Erik Taubeneck (eriktaubeneck) +* Frank Tackitt (kageurufu) +* Isaac Dickinson (SunDwarf) \ No newline at end of file diff --git a/flask_s3.py b/flask_s3.py index 138a696..68c4267 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -13,6 +13,8 @@ from flask import current_app logger = logging.getLogger('flask_s3') +import six + def hash_file(filename): """ Generate a hash for the contents of a file @@ -72,13 +74,13 @@ def url_for(endpoint, **values): def _bp_static_url(blueprint): """ builds the absolute url path for a blueprint's static folder """ - u = u'%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '') + u = six.u('%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '')) return u def _gather_files(app, hidden): """ Gets all files in static folders and returns in dict.""" - dirs = [(unicode(app.static_folder), app.static_url_path)] + dirs = [(six.u(app.static_folder), app.static_url_path)] if hasattr(app, 'blueprints'): blueprints = app.blueprints.values() bp_details = lambda x: (x.static_folder, _bp_static_url(x)) @@ -118,7 +120,7 @@ def _static_folder_path(static_url, static_folder, static_asset): (static_asset, static_folder)) rel_asset = static_asset[len(static_folder):] # Now bolt the static url path and the relative asset location together - return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')) + return six.u('%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/'))) def _write_files(s3, app, static_url_loc, static_folder, files, bucket, @@ -157,7 +159,7 @@ def _write_files(s3, app, static_url_loc, static_folder, files, bucket, def _upload_files(s3, app, files_, bucket, hashes=None): new_hashes = [] - for (static_folder, static_url), names in files_.iteritems(): + for (static_folder, static_url), names in six.iteritems(files_): new_hashes.extend(_write_files(s3, app, static_url, static_folder, names, bucket, hashes=hashes)) return new_hashes diff --git a/setup.py b/setup.py index 35daad5..7ae1560 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,8 @@ setup( platforms='any', install_requires=[ 'Flask', - 'Boto3>=1.1.1' + 'Boto3>=1.1.1', + 'six' ], tests_require=['nose', 'mock'], classifiers=[ @@ -33,5 +34,6 @@ setup( 'Programming Language :: Python', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' - ] + ], + test_suite = 'nose.collector' ) diff --git a/tests/test_flask_static.py b/test_flask_static.py similarity index 86% rename from tests/test_flask_static.py rename to test_flask_static.py index fde9ae1..76836cc 100644 --- a/tests/test_flask_static.py +++ b/test_flask_static.py @@ -3,8 +3,12 @@ import ntpath import tempfile import os -from mock import Mock, patch, call, mock_open +try: + from unittest.mock import Mock, patch, call, mock_open +except ImportError: + from mock import Mock, patch, call, mock_open from flask import Flask, render_template_string, Blueprint +import six import flask_s3 from flask_s3 import FlaskS3 @@ -67,7 +71,11 @@ class UrlTests(unittest.TestCase): def client_get(self, ufs): FlaskS3(self.app) client = self.app.test_client() - return client.get('/%s' % ufs) + import six + if six.PY3: + return client.get('/%s' % ufs) + elif six.PY2: + return client.get('/{}'.format(ufs)) def test_required_config(self): """ @@ -89,11 +97,11 @@ class UrlTests(unittest.TestCase): Tests that correct url formed for static asset in self.app. """ # non static endpoint url_for in template - self.assertEquals(self.client_get('').data, '/') + self.assertEquals(self.client_get('').data, six.b('/')) # static endpoint url_for in template ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_debug(self): """Tests Flask-S3 behaviour in debug mode.""" @@ -101,7 +109,7 @@ class UrlTests(unittest.TestCase): # static endpoint url_for in template ufs = "{{url_for('static', filename='bah.js')}}" exp = '/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_debug_override(self): """Tests Flask-S3 behavior in debug mode with USE_S3_DEBUG turned on.""" @@ -109,7 +117,7 @@ class UrlTests(unittest.TestCase): self.app.config['USE_S3_DEBUG'] = True ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_blueprint(self): """ @@ -118,26 +126,26 @@ class UrlTests(unittest.TestCase): # static endpoint url_for in template ufs = "{{url_for('admin.static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/admin-static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_cdn_domain(self): self.app.config['S3_CDN_DOMAIN'] = 'foo.cloudfront.net' ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.cloudfront.net/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_url_style_path(self): """Tests that the URL returned uses the path style.""" self.app.config['S3_URL_STYLE'] = 'path' ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://s3.amazonaws.com/foo/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_url_style_invalid(self): """Tests that an exception is raised for invalid URL styles.""" self.app.config['S3_URL_STYLE'] = 'balderdash' ufs = "{{url_for('static', filename='bah.js')}}" - self.assertRaises(ValueError, self.client_get, ufs) + self.assertRaises(ValueError, self.client_get, six.b(ufs)) class S3Tests(unittest.TestCase): @@ -159,7 +167,7 @@ class S3Tests(unittest.TestCase): Mock(static_url_path=None, url_prefix='/pref'), Mock(static_url_path='/b/bar', url_prefix='/pref'), Mock(static_url_path=None, url_prefix=None)] - expected = [u'/foo', u'/pref', u'/pref/b/bar', u''] + expected = [six.u('/foo'), six.u('/pref'), six.u('/pref/b/bar'), six.u('')] self.assertEquals(expected, [flask_s3._bp_static_url(x) for x in bps]) @patch('os.walk') @@ -183,14 +191,14 @@ class S3Tests(unittest.TestCase): os_mock.side_effect = dirs.get path_mock.return_value = True - expected = {('/home/bar', u'/a/bar'): ['/home/bar/b'], - ('/home/zoo', u'/b/bar'): ['/home/zoo/c', + expected = {('/home/bar', six.u('/a/bar')): ['/home/bar/b'], + ('/home/zoo', six.u('/b/bar')): ['/home/zoo/c', '/home/zoo/foo/d', '/home/zoo/foo/e']} actual = flask_s3._gather_files(self.app, False) self.assertEqual(expected, actual) - expected[('/home', u'/static')] = ['/home/.a'] + expected[('/home', six.u('/static'))] = ['/home/.a'] actual = flask_s3._gather_files(self.app, True) self.assertEqual(expected, actual) @@ -235,7 +243,7 @@ class S3Tests(unittest.TestCase): self.assertEquals(exp, actual) @patch('flask_s3.boto3') - @patch('__builtin__.open', mock_open(read_data='test')) + @patch("{}.open".format("builtins" if six.PY3 else "__builtins__"), mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' @@ -243,7 +251,7 @@ class S3Tests(unittest.TestCase): assets = ['/home/z/bar.css', '/home/z/foo.css'] exclude = ['/foo/static/foo.css', '/foo/static/foo/bar.css'] # we expect foo.css to be excluded and not uploaded - expected = [call(bucket=None, name=u'/foo/static/bar.css'), + expected = [call(bucket=None, name=six.u('/foo/static/bar.css')), call().set_metadata('Cache-Control', 'cache instruction'), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), @@ -272,8 +280,12 @@ class S3Tests(unittest.TestCase): for filename in filenames: # Write random data into files with open(filename, 'wb') as f: - data = str(data_iter.next()) - f.write(data) + if six.PY3: + data = str(data_iter) + f.write(data.encode()) + else: + data = str(data_iter.next()) + f.write(data) # We expect each file to be uploaded expected.append(call.put_object(ACL='public-read', @@ -294,7 +306,10 @@ class S3Tests(unittest.TestCase): # We now modify the second file with open(filenames[1], 'wb') as f: data = str(next(data_iter)) - f.write(data) + if six.PY2: + f.write(data) + else: + f.write(data.encode()) # We expect only this file to be uploaded expected.append(call.put_object(ACL='public-read', @@ -306,9 +321,9 @@ class S3Tests(unittest.TestCase): new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, hashes=dict(hashes)) - import pprint + #import pprint - pprint.pprint(zip(expected, key_mock.mock_calls)) + #pprint.pprint(zip(expected, key_mock.mock_calls)) self.assertEqual(expected, key_mock.mock_calls) def test_static_folder_path(self): @@ -316,8 +331,8 @@ class S3Tests(unittest.TestCase): inputs = [('/static', '/home/static', '/home/static/foo.css'), ('/foo/static', '/home/foo/s', '/home/foo/s/a/b.css'), ('/bar/', '/bar/', '/bar/s/a/b.css')] - expected = [u'/static/foo.css', u'/foo/static/a/b.css', - u'/bar/s/a/b.css'] + expected = [six.u('/static/foo.css'), six.u('/foo/static/a/b.css'), + six.u('/bar/s/a/b.css')] for i, e in zip(inputs, expected): self.assertEquals(e, flask_s3._static_folder_path(*i)) From cbd2140644641649f641cbe6201f731a7d01084c Mon Sep 17 00:00:00 2001 From: SunDwarf Date: Fri, 28 Aug 2015 19:52:41 +0100 Subject: [PATCH 5/6] Prevent url_for from building when app.testing is True/existing. Closes #23. --- .travis.yml | 6 +++++- docs/requirements.txt | 8 +++----- flask_s3.py | 2 ++ requirements.txt | 3 +++ test_flask_static.py | 6 +++++- 5 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 102e309..8b61183 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,11 @@ language: python python: - "2.7" + - "3.2" + - "3.3" + - "3.4" # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -r docs/requirements.txt --use-mirrors +install: pip install -r requirements.txt --use-mirrors # command to run tests, e.g. python setup.py test script: nosetests +sudo: false diff --git a/docs/requirements.txt b/docs/requirements.txt index 216a6f4..869cff6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,3 @@ -Flask==0.9 -Jinja2==2.6 -Werkzeug==0.8.3 -boto3==1.1.1 -wsgiref==0.1.2 +Flask2 +boto3 + diff --git a/flask_s3.py b/flask_s3.py index 68c4267..65f4f72 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -44,6 +44,8 @@ def url_for(endpoint, **values): of your templates. """ app = current_app + if app.config.get('TESTING', False) and not app.config.get('S3_OVERRIDE_TESTING', True): + return flask_url_for(endpoint, **values) if 'S3_BUCKET_NAME' not in app.config: raise ValueError("S3_BUCKET_NAME not found in app configuration.") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c3fe21 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +Flask +boto3 +six diff --git a/test_flask_static.py b/test_flask_static.py index 76836cc..843ecbd 100644 --- a/test_flask_static.py +++ b/test_flask_static.py @@ -2,6 +2,7 @@ import unittest import ntpath import tempfile import os +import sys try: from unittest.mock import Mock, patch, call, mock_open @@ -51,6 +52,7 @@ class UrlTests(unittest.TestCase): self.app.config['S3_USE_HTTPS'] = True self.app.config['S3_BUCKET_DOMAIN'] = 's3.amazonaws.com' self.app.config['S3_CDN_DOMAIN'] = '' + self.app.config['S3_OVERRIDE_TESTING'] = True @self.app.route('/') def a(url_for_string): @@ -242,8 +244,10 @@ class S3Tests(unittest.TestCase): actual = flask_s3._path_to_relative_url(in_) self.assertEquals(exp, actual) + @unittest.skipIf(sys.version_info < (3, 0), + "not supported in this version") @patch('flask_s3.boto3') - @patch("{}.open".format("builtins" if six.PY3 else "__builtins__"), mock_open(read_data='test')) + @patch("{}.open".format("builtins"), mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' From 8259049138e9e2336437328f55b813add2f2c9f8 Mon Sep 17 00:00:00 2001 From: SunDwarf Date: Fri, 28 Aug 2015 20:19:18 +0100 Subject: [PATCH 6/6] Force HTTPS by default. --- flask_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index ca3ce8a..b112b73 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -58,9 +58,9 @@ def url_for(endpoint, **values): raise ValueError("S3_BUCKET_NAME not found in app configuration.") if endpoint == 'static' or endpoint.endswith('.static'): - scheme = 'http' - if app.config['S3_USE_HTTPS']: - scheme = 'https' + scheme = 'https' + if app.config['S3_USE_HTTP']: + scheme = 'http' if app.config['S3_URL_STYLE'] == 'host': url_format = '%(bucket_name)s.%(bucket_domain)s'