diff --git a/.travis.yml b/.travis.yml index 102e309..8b61183 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,11 @@ language: python python: - "2.7" + - "3.2" + - "3.3" + - "3.4" # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -r docs/requirements.txt --use-mirrors +install: pip install -r requirements.txt --use-mirrors # command to run tests, e.g. python setup.py test script: nosetests +sudo: false diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e7de847..27fcfe2 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -5,3 +5,5 @@ Contributors * Rehan Dalal (rehandalal) * Hannes Ljungberg (hannseman) * Erik Taubeneck (eriktaubeneck) +* Frank Tackitt (kageurufu) +* Isaac Dickinson (SunDwarf) \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 7d01ae6..869cff6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,3 @@ -Flask==0.9 -Jinja2==2.6 -Werkzeug==0.8.3 -boto==2.29.1 -wsgiref==0.1.2 +Flask2 +boto3 + diff --git a/flask_s3.py b/flask_s3.py index e116605..b112b73 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -3,17 +3,26 @@ import logging import hashlib import json from collections import defaultdict +import re +import boto3 +import boto3.exceptions +from botocore.exceptions import ClientError from flask import url_for as flask_url_for from flask import current_app -from boto.s3.connection import S3Connection -from boto.s3 import connect_to_region -from boto.exception import S3CreateError, S3ResponseError -from boto.s3.key import Key logger = logging.getLogger('flask_s3') +import six + +def merge_two_dicts(x, y): + '''Given two dicts, merge them into a new dict as a shallow copy.''' + z = x.copy() + z.update(y) + return z + + def hash_file(filename): """ Generate a hash for the contents of a file @@ -43,13 +52,15 @@ def url_for(endpoint, **values): of your templates. """ app = current_app + if app.config.get('TESTING', False) and not app.config.get('S3_OVERRIDE_TESTING', True): + return flask_url_for(endpoint, **values) if 'S3_BUCKET_NAME' not in app.config: raise ValueError("S3_BUCKET_NAME not found in app configuration.") if endpoint == 'static' or endpoint.endswith('.static'): - scheme = 'http' - if app.config['S3_USE_HTTPS']: - scheme = 'https' + scheme = 'https' + if app.config['S3_USE_HTTP']: + scheme = 'http' if app.config['S3_URL_STYLE'] == 'host': url_format = '%(bucket_name)s.%(bucket_domain)s' @@ -73,13 +84,13 @@ def url_for(endpoint, **values): def _bp_static_url(blueprint): """ builds the absolute url path for a blueprint's static folder """ - u = u'%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '') + u = six.u('%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '')) return u def _gather_files(app, hidden): """ Gets all files in static folders and returns in dict.""" - dirs = [(unicode(app.static_folder), app.static_url_path)] + dirs = [(six.u(app.static_folder), app.static_url_path)] if hasattr(app, 'blueprints'): blueprints = app.blueprints.values() bp_details = lambda x: (x.static_folder, _bp_static_url(x)) @@ -119,40 +130,34 @@ def _static_folder_path(static_url, static_folder, static_asset): (static_asset, static_folder)) rel_asset = static_asset[len(static_folder):] # Now bolt the static url path and the relative asset location together - return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')) + return six.u('%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/'))) -def _write_files(app, static_url_loc, static_folder, files, bucket, +def _write_files(s3, app, static_url_loc, static_folder, files, bucket, ex_keys=None, hashes=None): """ Writes all the files inside a static folder to S3. """ new_hashes = [] static_folder_rel = _path_to_relative_url(static_folder) for file_path in files: asset_loc = _path_to_relative_url(file_path) - key_name = _static_folder_path(static_url_loc, static_folder_rel, + full_key_name = _static_folder_path(static_url_loc, static_folder_rel, asset_loc) + key_name = full_key_name.lstrip("/") msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) exclude = False if app.config.get('S3_ONLY_MODIFIED', False): file_hash = hash_file(file_path) - new_hashes.append((key_name, file_hash)) + new_hashes.append((full_key_name, file_hash)) - if hashes and hashes.get(key_name, None) == file_hash: + if hashes and hashes.get(full_key_name, None) == file_hash: exclude = True - if ex_keys and key_name in ex_keys or exclude: + if ex_keys and full_key_name in ex_keys or exclude: logger.debug("%s excluded from upload" % key_name) else: - k = Key(bucket=bucket, name=key_name) - - # Set custom headers - headers = app.config.get('S3_HEADERS') - if headers: - for header, value in headers.iteritems(): - k.set_metadata(header, value) - + h = {} # Set more custom headers if the filepath matches certain # configured regular expressions. filepath_headers = app.config.get('S3_FILEPATH_HEADERS') @@ -160,18 +165,25 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, for filepath_regex, headers in filepath_headers.iteritems(): if re.search(filepath_regex, file_path): for header, value in headers.iteritems(): - k.set_metadata(header, value) + h[header] = value + + with open(file_path) as fp: + s3.put_object(Bucket=bucket, + Key=key_name, + Body=fp.read(), + ACL="public-read", + Metadata=merge_two_dicts(app.config['S3_HEADERS'], h)) + + - k.set_contents_from_filename(file_path) - k.make_public() return new_hashes -def _upload_files(app, files_, bucket, hashes=None): +def _upload_files(s3, app, files_, bucket, hashes=None): new_hashes = [] - for (static_folder, static_url), names in files_.iteritems(): - new_hashes.extend(_write_files(app, static_url, static_folder, names, + for (static_folder, static_url), names in six.iteritems(files_): + new_hashes.extend(_write_files(s3, app, static_url, static_folder, names, bucket, hashes=hashes)) return new_hashes @@ -240,45 +252,42 @@ def create_all(app, user=None, password=None, bucket_name=None, logger.debug("All valid files: %s" % all_files) # connect to s3 - if not location: - conn = S3Connection(user, password) # (default region) - else: - conn = connect_to_region(location, - aws_access_key_id=user, - aws_secret_access_key=password) + s3 = boto3.client("s3", + region_name=location or None, + aws_access_key_id=user, + aws_secret_access_key=password) # get_or_create bucket try: - try: - bucket = conn.create_bucket(bucket_name) - except S3CreateError as e: - if e.error_code == u'BucketAlreadyOwnedByYou': - bucket = conn.get_bucket(bucket_name) - else: - raise e + s3.head_bucket(Bucket=bucket_name) + except ClientError as e: + if int(e.response['Error']['Code']) == 404: + # Create the bucket + bucket = s3.create_bucket(Bucket=bucket_name) + else: + raise - bucket.make_public(recursive=False) - except S3CreateError as e: - raise e + s3.put_bucket_acl(Bucket=bucket_name, ACL='public-read') if app.config['S3_ONLY_MODIFIED']: try: - hashes = json.loads( - Key(bucket=bucket, - name=".file-hashes").get_contents_as_string()) - except S3ResponseError as e: + hashes_object = s3.get_object(Bucket=bucket_name, Key='.file-hashes') + hashes = json.loads(str(hashes_object['Body'].read())) + except ClientError as e: logger.warn("No file hashes found: %s" % e) hashes = None - new_hashes = _upload_files(app, all_files, bucket, hashes=hashes) + new_hashes = _upload_files(s3, app, all_files, bucket_name, hashes=hashes) try: - k = Key(bucket=bucket, name=".file-hashes") - k.set_contents_from_string(json.dumps(dict(new_hashes))) - except S3ResponseError as e: + s3.put_object(Bucket=bucket_name, + Key='.file-hashes', + Body=json.dumps(dict(new_hashes)), + ACL='private') + except boto3.exceptions.S3UploadFailedError as e: logger.warn("Unable to upload file hashes: %s" % e) else: - _upload_files(app, all_files, bucket) + _upload_files(s3, app, all_files, bucket_name) class FlaskS3(object): @@ -292,6 +301,7 @@ class FlaskS3(object): :param app: optional :class:`flask.Flask` application object :type app: :class:`flask.Flask` or None """ + def __init__(self, app=None): if app is not None: self.init_app(app) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1c3fe21 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +Flask +boto3 +six diff --git a/setup.py b/setup.py index dfa4e22..7ae1560 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup setup( name='Flask-S3', - version='0.1.7', + version='0.2.0', url='http://github.com/e-dard/flask-s3', license='WTFPL', author='Edward Robinson', @@ -22,7 +22,8 @@ setup( platforms='any', install_requires=[ 'Flask', - 'Boto>=2.5.2' + 'Boto3>=1.1.1', + 'six' ], tests_require=['nose', 'mock'], classifiers=[ @@ -33,5 +34,6 @@ setup( 'Programming Language :: Python', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' - ] + ], + test_suite = 'nose.collector' ) diff --git a/tests/test_flask_static.py b/test_flask_static.py similarity index 75% rename from tests/test_flask_static.py rename to test_flask_static.py index 152e34d..843ecbd 100644 --- a/tests/test_flask_static.py +++ b/test_flask_static.py @@ -1,11 +1,15 @@ import unittest import ntpath import tempfile -import os.path import os +import sys -from mock import Mock, patch, call +try: + from unittest.mock import Mock, patch, call, mock_open +except ImportError: + from mock import Mock, patch, call, mock_open from flask import Flask, render_template_string, Blueprint +import six import flask_s3 from flask_s3 import FlaskS3 @@ -15,6 +19,7 @@ class FlaskStaticTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) self.app.testing = True + @self.app.route('/') def a(url_for_string): return render_template_string(url_for_string) @@ -47,6 +52,7 @@ class UrlTests(unittest.TestCase): self.app.config['S3_USE_HTTPS'] = True self.app.config['S3_BUCKET_DOMAIN'] = 's3.amazonaws.com' self.app.config['S3_CDN_DOMAIN'] = '' + self.app.config['S3_OVERRIDE_TESTING'] = True @self.app.route('/') def a(url_for_string): @@ -57,16 +63,21 @@ class UrlTests(unittest.TestCase): return render_template_string("{{url_for('b')}}") bp = Blueprint('admin', __name__, static_folder='admin-static') + @bp.route('/') def c(): return render_template_string("{{url_for('b')}}") - self.app.register_blueprint(bp) + self.app.register_blueprint(bp) def client_get(self, ufs): FlaskS3(self.app) client = self.app.test_client() - return client.get('/%s' % ufs) + import six + if six.PY3: + return client.get('/%s' % ufs) + elif six.PY2: + return client.get('/{}'.format(ufs)) def test_required_config(self): """ @@ -88,11 +99,11 @@ class UrlTests(unittest.TestCase): Tests that correct url formed for static asset in self.app. """ # non static endpoint url_for in template - self.assertEquals(self.client_get('').data, '/') + self.assertEquals(self.client_get('').data, six.b('/')) # static endpoint url_for in template ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_debug(self): """Tests Flask-S3 behaviour in debug mode.""" @@ -100,7 +111,7 @@ class UrlTests(unittest.TestCase): # static endpoint url_for in template ufs = "{{url_for('static', filename='bah.js')}}" exp = '/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_debug_override(self): """Tests Flask-S3 behavior in debug mode with USE_S3_DEBUG turned on.""" @@ -108,7 +119,7 @@ class UrlTests(unittest.TestCase): self.app.config['USE_S3_DEBUG'] = True ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_blueprint(self): """ @@ -117,30 +128,29 @@ class UrlTests(unittest.TestCase): # static endpoint url_for in template ufs = "{{url_for('admin.static', filename='bah.js')}}" exp = 'https://foo.s3.amazonaws.com/admin-static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_cdn_domain(self): self.app.config['S3_CDN_DOMAIN'] = 'foo.cloudfront.net' ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://foo.cloudfront.net/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_url_style_path(self): """Tests that the URL returned uses the path style.""" self.app.config['S3_URL_STYLE'] = 'path' ufs = "{{url_for('static', filename='bah.js')}}" exp = 'https://s3.amazonaws.com/foo/static/bah.js' - self.assertEquals(self.client_get(ufs).data, exp) + self.assertEquals(self.client_get(ufs).data, six.b(exp)) def test_url_for_url_style_invalid(self): """Tests that an exception is raised for invalid URL styles.""" self.app.config['S3_URL_STYLE'] = 'balderdash' ufs = "{{url_for('static', filename='bah.js')}}" - self.assertRaises(ValueError, self.client_get, ufs) + self.assertRaises(ValueError, self.client_get, six.b(ufs)) class S3Tests(unittest.TestCase): - def setUp(self): self.app = Flask(__name__) self.app.testing = True @@ -159,7 +169,7 @@ class S3Tests(unittest.TestCase): Mock(static_url_path=None, url_prefix='/pref'), Mock(static_url_path='/b/bar', url_prefix='/pref'), Mock(static_url_path=None, url_prefix=None)] - expected = [u'/foo', u'/pref', u'/pref/b/bar', u''] + expected = [six.u('/foo'), six.u('/pref'), six.u('/pref/b/bar'), six.u('')] self.assertEquals(expected, [flask_s3._bp_static_url(x) for x in bps]) @patch('os.walk') @@ -183,14 +193,14 @@ class S3Tests(unittest.TestCase): os_mock.side_effect = dirs.get path_mock.return_value = True - expected = {('/home/bar', u'/a/bar'): ['/home/bar/b'], - ('/home/zoo', u'/b/bar'): ['/home/zoo/c', + expected = {('/home/bar', six.u('/a/bar')): ['/home/bar/b'], + ('/home/zoo', six.u('/b/bar')): ['/home/zoo/c', '/home/zoo/foo/d', '/home/zoo/foo/e']} actual = flask_s3._gather_files(self.app, False) self.assertEqual(expected, actual) - expected[('/home', u'/static')] = ['/home/.a'] + expected[('/home', six.u('/static'))] = ['/home/.a'] actual = flask_s3._gather_files(self.app, True) self.assertEqual(expected, actual) @@ -234,7 +244,10 @@ class S3Tests(unittest.TestCase): actual = flask_s3._path_to_relative_url(in_) self.assertEquals(exp, actual) - @patch('flask_s3.Key') + @unittest.skipIf(sys.version_info < (3, 0), + "not supported in this version") + @patch('flask_s3.boto3') + @patch("{}.open".format("builtins"), mock_open(read_data='test')) def test__write_files(self, key_mock): """ Tests _write_files """ static_url_loc = '/foo/static' @@ -242,16 +255,16 @@ class S3Tests(unittest.TestCase): assets = ['/home/z/bar.css', '/home/z/foo.css'] exclude = ['/foo/static/foo.css', '/foo/static/foo/bar.css'] # we expect foo.css to be excluded and not uploaded - expected = [call(bucket=None, name=u'/foo/static/bar.css'), + expected = [call(bucket=None, name=six.u('/foo/static/bar.css')), call().set_metadata('Cache-Control', 'cache instruction'), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename('/home/z/bar.css')] - flask_s3._write_files(self.app, static_url_loc, static_folder, assets, + flask_s3._write_files(key_mock, self.app, static_url_loc, static_folder, assets, None, exclude) self.assertLessEqual(expected, key_mock.mock_calls) - @patch('flask_s3.Key') + @patch('flask_s3.boto3') def test__write_only_modified(self, key_mock): """ Test that we only upload files that have changed """ self.app.config['S3_ONLY_MODIFIED'] = True @@ -260,23 +273,35 @@ class S3Tests(unittest.TestCase): filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']] expected = [] + def IntIterator(): + i = 0 + while True: + i += 1 + yield i + + data_iter = IntIterator() for filename in filenames: # Write random data into files with open(filename, 'wb') as f: - f.write(os.urandom(1024)) + if six.PY3: + data = str(data_iter) + f.write(data.encode()) + else: + data = str(data_iter.next()) + f.write(data) # We expect each file to be uploaded - expected.extend([call(bucket=None, name=filename), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filename), - call().make_public()]) + expected.append(call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filename.lstrip("/"), + Body=data)) files = {(static_url_loc, static_folder): filenames} - hashes = flask_s3._upload_files(self.app, files, None) + hashes = flask_s3._upload_files(key_mock, self.app, files, None) # All files are uploaded and hashes are returned self.assertLessEqual(expected, key_mock.mock_calls) @@ -284,19 +309,25 @@ class S3Tests(unittest.TestCase): # We now modify the second file with open(filenames[1], 'wb') as f: - f.write(os.urandom(1024)) + data = str(next(data_iter)) + if six.PY2: + f.write(data) + else: + f.write(data.encode()) # We expect only this file to be uploaded - expected.extend([call(bucket=None, name=filenames[1]), - call().set_metadata('Expires', - 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Content-Encoding', 'gzip'), - call().set_contents_from_filename(filenames[1]), - call().make_public()]) + expected.append(call.put_object(ACL='public-read', + Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip'}, + Bucket=None, + Key=filenames[1].lstrip("/"), + Body=data)) - new_hashes = flask_s3._upload_files(self.app, files, None, - hashes=dict(hashes)) + new_hashes = flask_s3._upload_files(key_mock, self.app, files, None, + hashes=dict(hashes)) + #import pprint + #pprint.pprint(zip(expected, key_mock.mock_calls)) self.assertEqual(expected, key_mock.mock_calls) def test_static_folder_path(self): @@ -304,10 +335,11 @@ class S3Tests(unittest.TestCase): inputs = [('/static', '/home/static', '/home/static/foo.css'), ('/foo/static', '/home/foo/s', '/home/foo/s/a/b.css'), ('/bar/', '/bar/', '/bar/s/a/b.css')] - expected = [u'/static/foo.css', u'/foo/static/a/b.css', - u'/bar/s/a/b.css'] + expected = [six.u('/static/foo.css'), six.u('/foo/static/a/b.css'), + six.u('/bar/s/a/b.css')] for i, e in zip(inputs, expected): self.assertEquals(e, flask_s3._static_folder_path(*i)) + if __name__ == '__main__': unittest.main()