wassname/flask-s3
1
0
Fork 0
mirror of https://github.com/wassname/flask-s3.git synced 2026-09-10 12:00:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

I don't even know

Browse Source
This commit is contained in:
SunDwarf
2015-08-28 20:20:31 +01:00
parent ae2f3fbb49 8259049138
commit 1f9cf2a2c9
7 changed files with 154 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+5 -1
View File
@@ -1,7 +1,11 @@
language: python
python:
- "2.7"
- "3.2"
- "3.3"
- "3.4"
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install -r docs/requirements.txt --use-mirrors
install: pip install -r requirements.txt --use-mirrors
# command to run tests, e.g. python setup.py test
script: nosetests
sudo: false
CONTRIBUTORS
+2
View File
@@ -5,3 +5,5 @@ Contributors
* Rehan Dalal (rehandalal)
* Hannes Ljungberg (hannseman)
* Erik Taubeneck (eriktaubeneck)
* Frank Tackitt (kageurufu)
* Isaac Dickinson (SunDwarf)
docs/requirements.txt
+3 -5
View File
@@ -1,5 +1,3 @@
Flask==0.9
Jinja2==2.6
Werkzeug==0.8.3
boto==2.29.1
wsgiref==0.1.2
Flask2
boto3
flask_s3.py
+64 -54
View File
@@ -3,17 +3,26 @@ import logging
import hashlib
import json
from collections import defaultdict
import re
import boto3
import boto3.exceptions
from botocore.exceptions import ClientError
from flask import url_for as flask_url_for
from flask import current_app
from boto.s3.connection import S3Connection
from boto.s3 import connect_to_region
from boto.exception import S3CreateError, S3ResponseError
from boto.s3.key import Key
logger = logging.getLogger('flask_s3')
import six
def merge_two_dicts(x, y):
'''Given two dicts, merge them into a new dict as a shallow copy.'''
z = x.copy()
z.update(y)
return z
def hash_file(filename):
"""
Generate a hash for the contents of a file
@@ -43,13 +52,15 @@ def url_for(endpoint, **values):
of your templates.
"""
app = current_app
if app.config.get('TESTING', False) and not app.config.get('S3_OVERRIDE_TESTING', True):
return flask_url_for(endpoint, **values)
if 'S3_BUCKET_NAME' not in app.config:
raise ValueError("S3_BUCKET_NAME not found in app configuration.")
if endpoint == 'static' or endpoint.endswith('.static'):
scheme = 'http'
if app.config['S3_USE_HTTPS']:
scheme = 'https'
scheme = 'https'
if app.config['S3_USE_HTTP']:
scheme = 'http'
if app.config['S3_URL_STYLE'] == 'host':
url_format = '%(bucket_name)s.%(bucket_domain)s'
@@ -73,13 +84,13 @@ def url_for(endpoint, **values):
def _bp_static_url(blueprint):
""" builds the absolute url path for a blueprint's static folder """
u = u'%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '')
u = six.u('%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or ''))
return u
def _gather_files(app, hidden):
""" Gets all files in static folders and returns in dict."""
dirs = [(unicode(app.static_folder), app.static_url_path)]
dirs = [(six.u(app.static_folder), app.static_url_path)]
if hasattr(app, 'blueprints'):
blueprints = app.blueprints.values()
bp_details = lambda x: (x.static_folder, _bp_static_url(x))
@@ -119,40 +130,34 @@ def _static_folder_path(static_url, static_folder, static_asset):
(static_asset, static_folder))
rel_asset = static_asset[len(static_folder):]
# Now bolt the static url path and the relative asset location together
return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/'))
return six.u('%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')))
def _write_files(app, static_url_loc, static_folder, files, bucket,
def _write_files(s3, app, static_url_loc, static_folder, files, bucket,
ex_keys=None, hashes=None):
""" Writes all the files inside a static folder to S3. """
new_hashes = []
static_folder_rel = _path_to_relative_url(static_folder)
for file_path in files:
asset_loc = _path_to_relative_url(file_path)
key_name = _static_folder_path(static_url_loc, static_folder_rel,
full_key_name = _static_folder_path(static_url_loc, static_folder_rel,
asset_loc)
key_name = full_key_name.lstrip("/")
msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name)
logger.debug(msg)
exclude = False
if app.config.get('S3_ONLY_MODIFIED', False):
file_hash = hash_file(file_path)
new_hashes.append((key_name, file_hash))
new_hashes.append((full_key_name, file_hash))
if hashes and hashes.get(key_name, None) == file_hash:
if hashes and hashes.get(full_key_name, None) == file_hash:
exclude = True
if ex_keys and key_name in ex_keys or exclude:
if ex_keys and full_key_name in ex_keys or exclude:
logger.debug("%s excluded from upload" % key_name)
else:
k = Key(bucket=bucket, name=key_name)
# Set custom headers
headers = app.config.get('S3_HEADERS')
if headers:
for header, value in headers.iteritems():
k.set_metadata(header, value)
h = {}
# Set more custom headers if the filepath matches certain
# configured regular expressions.
filepath_headers = app.config.get('S3_FILEPATH_HEADERS')
@@ -160,18 +165,25 @@ def _write_files(app, static_url_loc, static_folder, files, bucket,
for filepath_regex, headers in filepath_headers.iteritems():
if re.search(filepath_regex, file_path):
for header, value in headers.iteritems():
k.set_metadata(header, value)
h[header] = value
with open(file_path) as fp:
s3.put_object(Bucket=bucket,
Key=key_name,
Body=fp.read(),
ACL="public-read",
Metadata=merge_two_dicts(app.config['S3_HEADERS'], h))
k.set_contents_from_filename(file_path)
k.make_public()
return new_hashes
def _upload_files(app, files_, bucket, hashes=None):
def _upload_files(s3, app, files_, bucket, hashes=None):
new_hashes = []
for (static_folder, static_url), names in files_.iteritems():
new_hashes.extend(_write_files(app, static_url, static_folder, names,
for (static_folder, static_url), names in six.iteritems(files_):
new_hashes.extend(_write_files(s3, app, static_url, static_folder, names,
bucket, hashes=hashes))
return new_hashes
@@ -240,45 +252,42 @@ def create_all(app, user=None, password=None, bucket_name=None,
logger.debug("All valid files: %s" % all_files)
# connect to s3
if not location:
conn = S3Connection(user, password) # (default region)
else:
conn = connect_to_region(location,
aws_access_key_id=user,
aws_secret_access_key=password)
s3 = boto3.client("s3",
region_name=location or None,
aws_access_key_id=user,
aws_secret_access_key=password)
# get_or_create bucket
try:
try:
bucket = conn.create_bucket(bucket_name)
except S3CreateError as e:
if e.error_code == u'BucketAlreadyOwnedByYou':
bucket = conn.get_bucket(bucket_name)
else:
raise e
s3.head_bucket(Bucket=bucket_name)
except ClientError as e:
if int(e.response['Error']['Code']) == 404:
# Create the bucket
bucket = s3.create_bucket(Bucket=bucket_name)
else:
raise
bucket.make_public(recursive=False)
except S3CreateError as e:
raise e
s3.put_bucket_acl(Bucket=bucket_name, ACL='public-read')
if app.config['S3_ONLY_MODIFIED']:
try:
hashes = json.loads(
Key(bucket=bucket,
name=".file-hashes").get_contents_as_string())
except S3ResponseError as e:
hashes_object = s3.get_object(Bucket=bucket_name, Key='.file-hashes')
hashes = json.loads(str(hashes_object['Body'].read()))
except ClientError as e:
logger.warn("No file hashes found: %s" % e)
hashes = None
new_hashes = _upload_files(app, all_files, bucket, hashes=hashes)
new_hashes = _upload_files(s3, app, all_files, bucket_name, hashes=hashes)
try:
k = Key(bucket=bucket, name=".file-hashes")
k.set_contents_from_string(json.dumps(dict(new_hashes)))
except S3ResponseError as e:
s3.put_object(Bucket=bucket_name,
Key='.file-hashes',
Body=json.dumps(dict(new_hashes)),
ACL='private')
except boto3.exceptions.S3UploadFailedError as e:
logger.warn("Unable to upload file hashes: %s" % e)
else:
_upload_files(app, all_files, bucket)
_upload_files(s3, app, all_files, bucket_name)
class FlaskS3(object):
@@ -292,6 +301,7 @@ class FlaskS3(object):
:param app: optional :class:`flask.Flask` application object
:type app: :class:`flask.Flask` or None
"""
def __init__(self, app=None):
if app is not None:
self.init_app(app)
requirements.txt
+3
View File
@@ -0,0 +1,3 @@
Flask
boto3
six
setup.py
+5 -3
View File
@@ -9,7 +9,7 @@ from setuptools import setup
setup(
name='Flask-S3',
version='0.1.7',
version='0.2.0',
url='http://github.com/e-dard/flask-s3',
license='WTFPL',
author='Edward Robinson',
@@ -22,7 +22,8 @@ setup(
platforms='any',
install_requires=[
'Flask',
'Boto>=2.5.2'
'Boto3>=1.1.1',
'six'
],
tests_require=['nose', 'mock'],
classifiers=[
@@ -33,5 +34,6 @@ setup(
'Programming Language :: Python',
'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
'Topic :: Software Development :: Libraries :: Python Modules'
]
],
test_suite = 'nose.collector'
)
tests/test_flask_static.py → test_flask_static.py
+72 -40
View File
@@ -1,11 +1,15 @@
import unittest
import ntpath
import tempfile
import os.path
import os
import sys
from mock import Mock, patch, call
try:
from unittest.mock import Mock, patch, call, mock_open
except ImportError:
from mock import Mock, patch, call, mock_open
from flask import Flask, render_template_string, Blueprint
import six
import flask_s3
from flask_s3 import FlaskS3
@@ -15,6 +19,7 @@ class FlaskStaticTest(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
self.app.testing = True
@self.app.route('/<url_for_string>')
def a(url_for_string):
return render_template_string(url_for_string)
@@ -47,6 +52,7 @@ class UrlTests(unittest.TestCase):
self.app.config['S3_USE_HTTPS'] = True
self.app.config['S3_BUCKET_DOMAIN'] = 's3.amazonaws.com'
self.app.config['S3_CDN_DOMAIN'] = ''
self.app.config['S3_OVERRIDE_TESTING'] = True
@self.app.route('/<url_for_string>')
def a(url_for_string):
@@ -57,16 +63,21 @@ class UrlTests(unittest.TestCase):
return render_template_string("{{url_for('b')}}")
bp = Blueprint('admin', __name__, static_folder='admin-static')
@bp.route('/<url_for_string>')
def c():
return render_template_string("{{url_for('b')}}")
self.app.register_blueprint(bp)
self.app.register_blueprint(bp)
def client_get(self, ufs):
FlaskS3(self.app)
client = self.app.test_client()
return client.get('/%s' % ufs)
import six
if six.PY3:
return client.get('/%s' % ufs)
elif six.PY2:
return client.get('/{}'.format(ufs))
def test_required_config(self):
"""
@@ -88,11 +99,11 @@ class UrlTests(unittest.TestCase):
Tests that correct url formed for static asset in self.app.
"""
# non static endpoint url_for in template
self.assertEquals(self.client_get('').data, '/')
self.assertEquals(self.client_get('').data, six.b('/'))
# static endpoint url_for in template
ufs = "{{url_for('static', filename='bah.js')}}"
exp = 'https://foo.s3.amazonaws.com/static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_debug(self):
"""Tests Flask-S3 behaviour in debug mode."""
@@ -100,7 +111,7 @@ class UrlTests(unittest.TestCase):
# static endpoint url_for in template
ufs = "{{url_for('static', filename='bah.js')}}"
exp = '/static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_debug_override(self):
"""Tests Flask-S3 behavior in debug mode with USE_S3_DEBUG turned on."""
@@ -108,7 +119,7 @@ class UrlTests(unittest.TestCase):
self.app.config['USE_S3_DEBUG'] = True
ufs = "{{url_for('static', filename='bah.js')}}"
exp = 'https://foo.s3.amazonaws.com/static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_blueprint(self):
"""
@@ -117,30 +128,29 @@ class UrlTests(unittest.TestCase):
# static endpoint url_for in template
ufs = "{{url_for('admin.static', filename='bah.js')}}"
exp = 'https://foo.s3.amazonaws.com/admin-static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_cdn_domain(self):
self.app.config['S3_CDN_DOMAIN'] = 'foo.cloudfront.net'
ufs = "{{url_for('static', filename='bah.js')}}"
exp = 'https://foo.cloudfront.net/static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_url_style_path(self):
"""Tests that the URL returned uses the path style."""
self.app.config['S3_URL_STYLE'] = 'path'
ufs = "{{url_for('static', filename='bah.js')}}"
exp = 'https://s3.amazonaws.com/foo/static/bah.js'
self.assertEquals(self.client_get(ufs).data, exp)
self.assertEquals(self.client_get(ufs).data, six.b(exp))
def test_url_for_url_style_invalid(self):
"""Tests that an exception is raised for invalid URL styles."""
self.app.config['S3_URL_STYLE'] = 'balderdash'
ufs = "{{url_for('static', filename='bah.js')}}"
self.assertRaises(ValueError, self.client_get, ufs)
self.assertRaises(ValueError, self.client_get, six.b(ufs))
class S3Tests(unittest.TestCase):
def setUp(self):
self.app = Flask(__name__)
self.app.testing = True
@@ -159,7 +169,7 @@ class S3Tests(unittest.TestCase):
Mock(static_url_path=None, url_prefix='/pref'),
Mock(static_url_path='/b/bar', url_prefix='/pref'),
Mock(static_url_path=None, url_prefix=None)]
expected = [u'/foo', u'/pref', u'/pref/b/bar', u'']
expected = [six.u('/foo'), six.u('/pref'), six.u('/pref/b/bar'), six.u('')]
self.assertEquals(expected, [flask_s3._bp_static_url(x) for x in bps])
@patch('os.walk')
@@ -183,14 +193,14 @@ class S3Tests(unittest.TestCase):
os_mock.side_effect = dirs.get
path_mock.return_value = True
expected = {('/home/bar', u'/a/bar'): ['/home/bar/b'],
('/home/zoo', u'/b/bar'): ['/home/zoo/c',
expected = {('/home/bar', six.u('/a/bar')): ['/home/bar/b'],
('/home/zoo', six.u('/b/bar')): ['/home/zoo/c',
'/home/zoo/foo/d',
'/home/zoo/foo/e']}
actual = flask_s3._gather_files(self.app, False)
self.assertEqual(expected, actual)
expected[('/home', u'/static')] = ['/home/.a']
expected[('/home', six.u('/static'))] = ['/home/.a']
actual = flask_s3._gather_files(self.app, True)
self.assertEqual(expected, actual)
@@ -234,7 +244,10 @@ class S3Tests(unittest.TestCase):
actual = flask_s3._path_to_relative_url(in_)
self.assertEquals(exp, actual)
@patch('flask_s3.Key')
@unittest.skipIf(sys.version_info < (3, 0),
"not supported in this version")
@patch('flask_s3.boto3')
@patch("{}.open".format("builtins"), mock_open(read_data='test'))
def test__write_files(self, key_mock):
""" Tests _write_files """
static_url_loc = '/foo/static'
@@ -242,16 +255,16 @@ class S3Tests(unittest.TestCase):
assets = ['/home/z/bar.css', '/home/z/foo.css']
exclude = ['/foo/static/foo.css', '/foo/static/foo/bar.css']
# we expect foo.css to be excluded and not uploaded
expected = [call(bucket=None, name=u'/foo/static/bar.css'),
expected = [call(bucket=None, name=six.u('/foo/static/bar.css')),
call().set_metadata('Cache-Control', 'cache instruction'),
call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename('/home/z/bar.css')]
flask_s3._write_files(self.app, static_url_loc, static_folder, assets,
flask_s3._write_files(key_mock, self.app, static_url_loc, static_folder, assets,
None, exclude)
self.assertLessEqual(expected, key_mock.mock_calls)
@patch('flask_s3.Key')
@patch('flask_s3.boto3')
def test__write_only_modified(self, key_mock):
""" Test that we only upload files that have changed """
self.app.config['S3_ONLY_MODIFIED'] = True
@@ -260,23 +273,35 @@ class S3Tests(unittest.TestCase):
filenames = [os.path.join(static_folder, f) for f in ['foo.css', 'bar.css']]
expected = []
def IntIterator():
i = 0
while True:
i += 1
yield i
data_iter = IntIterator()
for filename in filenames:
# Write random data into files
with open(filename, 'wb') as f:
f.write(os.urandom(1024))
if six.PY3:
data = str(data_iter)
f.write(data.encode())
else:
data = str(data_iter.next())
f.write(data)
# We expect each file to be uploaded
expected.extend([call(bucket=None, name=filename),
call().set_metadata('Expires',
'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filename),
call().make_public()])
expected.append(call.put_object(ACL='public-read',
Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT',
'Content-Encoding': 'gzip'},
Bucket=None,
Key=filename.lstrip("/"),
Body=data))
files = {(static_url_loc, static_folder): filenames}
hashes = flask_s3._upload_files(self.app, files, None)
hashes = flask_s3._upload_files(key_mock, self.app, files, None)
# All files are uploaded and hashes are returned
self.assertLessEqual(expected, key_mock.mock_calls)
@@ -284,19 +309,25 @@ class S3Tests(unittest.TestCase):
# We now modify the second file
with open(filenames[1], 'wb') as f:
f.write(os.urandom(1024))
data = str(next(data_iter))
if six.PY2:
f.write(data)
else:
f.write(data.encode())
# We expect only this file to be uploaded
expected.extend([call(bucket=None, name=filenames[1]),
call().set_metadata('Expires',
'Thu, 31 Dec 2037 23:59:59 GMT'),
call().set_metadata('Content-Encoding', 'gzip'),
call().set_contents_from_filename(filenames[1]),
call().make_public()])
expected.append(call.put_object(ACL='public-read',
Metadata={'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT',
'Content-Encoding': 'gzip'},
Bucket=None,
Key=filenames[1].lstrip("/"),
Body=data))
new_hashes = flask_s3._upload_files(self.app, files, None,
hashes=dict(hashes))
new_hashes = flask_s3._upload_files(key_mock, self.app, files, None,
hashes=dict(hashes))
#import pprint
#pprint.pprint(zip(expected, key_mock.mock_calls))
self.assertEqual(expected, key_mock.mock_calls)
def test_static_folder_path(self):
@@ -304,10 +335,11 @@ class S3Tests(unittest.TestCase):
inputs = [('/static', '/home/static', '/home/static/foo.css'),
('/foo/static', '/home/foo/s', '/home/foo/s/a/b.css'),
('/bar/', '/bar/', '/bar/s/a/b.css')]
expected = [u'/static/foo.css', u'/foo/static/a/b.css',
u'/bar/s/a/b.css']
expected = [six.u('/static/foo.css'), six.u('/foo/static/a/b.css'),
six.u('/bar/s/a/b.css')]
for i, e in zip(inputs, expected):
self.assertEquals(e, flask_s3._static_folder_path(*i))
if __name__ == '__main__':
unittest.main()