From a5f9df5012d7ba247e0041b0907e6b0b58740c79 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Wed, 1 May 2013 14:05:34 +0200 Subject: [PATCH 1/4] Added support for setting custom headers for files uploaded to s3. --- .gitignore | 5 +++++ CONTRIBUTORS | 1 + docs/index.rst | 19 +++++++++++++------ flask_s3.py | 8 ++++---- tests/test_flask_static.py | 30 ++++++++++++++++++------------ 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 4930881..5672901 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ *.pyc +.DS_Store +*.egg +*.egg-info +dist +/.idea _build diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 3d0750f..96f3929 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,3 +3,4 @@ Contributors * Edward Robinson (e-dard) * Rehan Dalal (rehandalal) +* Hannes Ljungberg (hannseman) diff --git a/docs/index.rst b/docs/index.rst index 459ecde..2bf473e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -163,12 +163,6 @@ uploading assets to S3. **Default:** ``u's3.amazonaws.com'`` `S3_BUCKET_NAME` The desired name for your Amazon S3 bucket. Note: the name will be visible in all your assets' URLs. -`S3_CACHE_CONTROL` This sets the value of the Cache-Control header that - is set in the metadata when `S3_USE_CACHE_CONTRL` is - set to `True`. -`S3_USE_CACHE_CONTROL` Specifies whether or not to set the metadata for the - Cache-Control headers. - **Default:** `False` `S3_USE_HTTPS` Specifies whether or not to serve your assets stored in S3 over HTTPS. **Default:** `True` @@ -186,6 +180,19 @@ uploading assets to S3. **Note**: if `USE_S3` is set to `False` then templates will always include asset locations specified by `flask.url_for`. +`S3_HEADERS` Sets custom headers to be sent with each file to S3. + + .. code-block:: python + + S3_HEADERS = { + 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', + 'Cache-Control': 'max-age=86400', + } + + See http://developer.yahoo.com/performance/rules.html#expires + for more information. + + **Default:** `{}` =========================== =================================================== .. _debug: http://flask.pocoo.org/docs/config/#configuration-basics diff --git a/flask_s3.py b/flask_s3.py index c163c7d..03c4f3a 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -101,9 +101,9 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, logger.debug("%s excluded from upload" % key_name) else: k = Key(bucket=bucket, name=key_name) - if (app.config['S3_USE_CACHE_CONTROL'] and - 'S3_CACHE_CONTROL' in app.config): - k.set_metadata('Cache-Control', app.config['S3_CACHE_CONTROL']) + # Set custom headers + for header, value in app.config['S3_HEADERS'].items(): + k.set_metadata(header, value) k.set_contents_from_filename(file_path) k.make_public() @@ -211,7 +211,7 @@ class FlaskS3(object): ('USE_S3', True), ('USE_S3_DEBUG', False), ('S3_BUCKET_DOMAIN', 's3.amazonaws.com'), - ('S3_USE_CACHE_CONTROL', False)] + ('S3_HEADERS', {})] for k, v in defaults: app.config.setdefault(k, v) diff --git a/tests/test_flask_static.py b/tests/test_flask_static.py index 9cdf3fc..1c9803a 100644 --- a/tests/test_flask_static.py +++ b/tests/test_flask_static.py @@ -28,7 +28,7 @@ class FlaskStaticTest(unittest.TestCase): """ Tests configuration vars exist. """ FlaskS3(self.app) defaults = ('S3_USE_HTTPS', 'USE_S3', 'USE_S3_DEBUG', - 'S3_BUCKET_DOMAIN', 'S3_USE_CACHE_CONTROL') + 'S3_BUCKET_DOMAIN', 'S3_HEADERS') for default in defaults: self.assertIn(default, self.app.config) @@ -117,7 +117,11 @@ class S3Tests(unittest.TestCase): self.app.testing = True self.app.config['S3_BUCKET_NAME'] = 'foo' self.app.config['S3_USE_CACHE_CONTROL'] = True - self.app.config['S3_CACHE_CONTROL'] = 'cache instruction' + self.app.config['S3_HEADERS'] = { + 'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', + 'Content-Encoding': 'gzip', + 'Cache-Control': 'max-age=86400' + } def test__bp_static_url(self): """ Tests test__bp_static_url """ @@ -141,12 +145,12 @@ class S3Tests(unittest.TestCase): url_prefix=None) bp_c = Mock(static_folder=None) - self.app.blueprints = { 'a': bp_a, 'b': bp_b, 'c': bp_c} - dirs = { '/home': [('/home', None, ['.a'])], - '/home/bar': [('/home/bar', None, ['b'])], - '/home/zoo': [('/home/zoo', None, ['c']), - ('/home/zoo/foo', None, ['d', 'e'])] } - os_mock.side_effect=dirs.get + self.app.blueprints = {'a': bp_a, 'b': bp_b, 'c': bp_c} + dirs = {'/home': [('/home', None, ['.a'])], + '/home/bar': [('/home/bar', None, ['b'])], + '/home/zoo': [('/home/zoo', None, ['c']), + ('/home/zoo/foo', None, ['d', 'e'])]} + os_mock.side_effect = dirs.get path_mock.return_value = True expected = {('/home/bar', u'/a/bar'): ['/home/bar/b'], @@ -169,7 +173,7 @@ class S3Tests(unittest.TestCase): """ self.app.static_folder = '/foo' dirs = {'/foo': [('/foo', None, [])]} - os_mock.side_effect=dirs.get + os_mock.side_effect = dirs.get path_mock.return_value = True actual = flask_s3._gather_files(self.app, False) @@ -183,7 +187,7 @@ class S3Tests(unittest.TestCase): """ self.app.static_folder = '/bad' dirs = {'/bad': []} - os_mock.side_effect=dirs.get + os_mock.side_effect = dirs.get path_mock.return_value = False actual = flask_s3._gather_files(self.app, False) @@ -209,7 +213,9 @@ class S3Tests(unittest.TestCase): exclude = ['/foo/static/foo.css', '/foo/static/foo/bar.css'] # we expect foo.css to be excluded and not uploaded expected = [call(bucket=None, name=u'/foo/static/bar.css'), - call().set_metadata('Cache-Control', 'cache instruction'), + call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), + call().set_metadata('Cache-Control', 'max-age=86400'), + call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename('/home/z/bar.css')] flask_s3._write_files(self.app, static_url_loc, static_folder, assets, None, exclude) @@ -226,4 +232,4 @@ class S3Tests(unittest.TestCase): self.assertEquals(e, flask_s3._static_folder_path(*i)) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 8c0eff3f974c0e01979c8870e5a98adba1d54bf7 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Tue, 11 Jun 2013 00:52:04 +0200 Subject: [PATCH 2/4] Updates issue #7 - Brings back S3_USE_CACHE_CONTROL setting and fixes some documentation issues. --- docs/index.rst | 37 +++++++++++++++++++++++++------------ flask_s3.py | 32 ++++++++++++++++++++++---------- tests/test_flask_static.py | 8 ++++---- 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2bf473e..a95768c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -139,6 +139,22 @@ generated by Flask-S3 will look like the following: that ``mybucketname`` is the name of your S3 bucket, and you have chosen to have assets served over HTTPS. +Setting Custom HTTP Headers +~~~~~~~~~~~~~~~~~ + +To set custom HTTP headers on the files served from S3 specify what +headers you want to use with the `S3_HEADERS` option. + +.. code-block:: python + + S3_HEADERS = { + 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', + 'Cache-Control': 'max-age=86400', + } + +See `yahoo`_ more information on how to set good values for your headers. + +.. _yahoo: http://developer.yahoo.com/performance/rules.html#expires .. _settings: .. _configuration: @@ -147,7 +163,7 @@ Flask-S3 Options ---------------- Within your Flask application's settings you can provide the following -settings to control the behvaiour of Flask-S3. None of the settings are +settings to control the behaviour of Flask-S3. None of the settings are required, but if not present, some will need to be provided when uploading assets to S3. @@ -181,18 +197,15 @@ uploading assets to S3. templates will always include asset locations specified by `flask.url_for`. `S3_HEADERS` Sets custom headers to be sent with each file to S3. - - .. code-block:: python - - S3_HEADERS = { - 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', - 'Cache-Control': 'max-age=86400', - } - - See http://developer.yahoo.com/performance/rules.html#expires - for more information. - **Default:** `{}` +`S3_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. + This sets the value of the Cache-Control header that + is set in the metadata when `S3_USE_CACHE_CONTRL` is + set to `True`. +`S3_USE_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. + Specifies whether or not to set the metadata for the + Cache-Control headers. + **Default:** `False` =========================== =================================================== .. _debug: http://flask.pocoo.org/docs/config/#configuration-basics diff --git a/flask_s3.py b/flask_s3.py index 03c4f3a..043dbdb 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -10,6 +10,7 @@ from boto.s3.key import Key logger = logging.getLogger('flask_s3') + def url_for(endpoint, **values): """ Generates a URL to the given endpoint. @@ -27,25 +28,27 @@ def url_for(endpoint, **values): app = current_app if 'S3_BUCKET_NAME' not in app.config: raise ValueError("S3_BUCKET_NAME not found in app configuration.") - + if app.debug and not app.config['USE_S3_DEBUG']: return flask_url_for(endpoint, **values) - + if endpoint == 'static' or endpoint.endswith('.static'): scheme = 'http' if app.config['S3_USE_HTTPS']: scheme = 'https' - bucket_path = '%s.%s' % (app.config['S3_BUCKET_NAME'], + bucket_path = '%s.%s' % (app.config['S3_BUCKET_NAME'], app.config['S3_BUCKET_DOMAIN']) urls = app.url_map.bind(bucket_path, url_scheme=scheme) return urls.build(endpoint, values=values, force_external=True) return flask_url_for(endpoint, **values) + def _bp_static_url(blueprint): """ builds the absolute url path for a blueprint's static folder """ u = u'%s%s' % (blueprint.url_prefix or '', blueprint.static_url_path or '') return u + def _gather_files(app, hidden): """ Gets all files in static folders and returns in dict.""" dirs = [(unicode(app.static_folder), app.static_url_path)] @@ -53,7 +56,7 @@ def _gather_files(app, hidden): blueprints = app.blueprints.values() bp_details = lambda x: (x.static_folder, _bp_static_url(x)) dirs.extend([bp_details(x) for x in blueprints if x.static_folder]) - + valid_files = defaultdict(list) for static_folder, static_url_loc in dirs: if not os.path.isdir(static_folder): @@ -67,10 +70,12 @@ def _gather_files(app, hidden): valid_files[(static_folder, static_url_loc)].extend(files) return valid_files + def _path_to_relative_url(path): """ Converts a folder and filename into a ralative url path """ return os.path.splitdrive(path)[1].replace('\\', '/') + def _static_folder_path(static_url, static_folder, static_asset): """ Returns a path to a file based on the static folder, and not on the @@ -88,12 +93,13 @@ def _static_folder_path(static_url, static_folder, static_asset): # Now bolt the static url path and the relative asset location together return u'%s/%s' % (static_url.rstrip('/'), rel_asset.lstrip('/')) -def _write_files(app, static_url_loc, static_folder, files, bucket, + +def _write_files(app, static_url_loc, static_folder, files, bucket, ex_keys=None): """ Writes all the files inside a static folder to S3. """ for file_path in files: asset_loc = _path_to_relative_url(file_path) - key_name = _static_folder_path(static_url_loc, static_folder, + key_name = _static_folder_path(static_url_loc, static_folder, asset_loc) msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) @@ -101,17 +107,22 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, logger.debug("%s excluded from upload" % key_name) else: k = Key(bucket=bucket, name=key_name) + if (app.config['S3_USE_CACHE_CONTROL'] and + 'S3_CACHE_CONTROL' in app.config): + k.set_metadata('Cache-Control', app.config['S3_CACHE_CONTROL']) # Set custom headers - for header, value in app.config['S3_HEADERS'].items(): + for header, value in app.config['S3_HEADERS'].iteritems(): k.set_metadata(header, value) k.set_contents_from_filename(file_path) k.make_public() + def _upload_files(app, files_, bucket): for (static_folder, static_url), names in files_.iteritems(): _write_files(app, static_url, static_folder, names, bucket) -def create_all(app, user=None, password=None, bucket_name=None, + +def create_all(app, user=None, password=None, bucket_name=None, location='', include_hidden=False): """ Uploads of the static assets associated with a Flask application to @@ -207,10 +218,11 @@ class FlaskS3(object): :param app: the :class:`flask.Flask` application object. """ - defaults = [('S3_USE_HTTPS', True), - ('USE_S3', True), + defaults = [('S3_USE_HTTPS', True), + ('USE_S3', True), ('USE_S3_DEBUG', False), ('S3_BUCKET_DOMAIN', 's3.amazonaws.com'), + ('S3_USE_CACHE_CONTROL', False), ('S3_HEADERS', {})] for k, v in defaults: app.config.setdefault(k, v) diff --git a/tests/test_flask_static.py b/tests/test_flask_static.py index 1c9803a..a8db00b 100644 --- a/tests/test_flask_static.py +++ b/tests/test_flask_static.py @@ -7,6 +7,7 @@ from flask import Flask, render_template_string, Blueprint import flask_s3 from flask_s3 import FlaskS3 + class FlaskStaticTest(unittest.TestCase): def setUp(self): self.app = Flask(__name__) @@ -28,12 +29,11 @@ class FlaskStaticTest(unittest.TestCase): """ Tests configuration vars exist. """ FlaskS3(self.app) defaults = ('S3_USE_HTTPS', 'USE_S3', 'USE_S3_DEBUG', - 'S3_BUCKET_DOMAIN', 'S3_HEADERS') + 'S3_BUCKET_DOMAIN', 'S3_USE_CACHE_CONTROL', 'S3_HEADERS') for default in defaults: self.assertIn(default, self.app.config) - class UrlTests(unittest.TestCase): def setUp(self): self.app = Flask(__name__) @@ -117,10 +117,10 @@ class S3Tests(unittest.TestCase): self.app.testing = True self.app.config['S3_BUCKET_NAME'] = 'foo' self.app.config['S3_USE_CACHE_CONTROL'] = True + self.app.config['S3_CACHE_CONTROL'] = 'cache instruction' self.app.config['S3_HEADERS'] = { 'Expires': 'Thu, 31 Dec 2037 23:59:59 GMT', 'Content-Encoding': 'gzip', - 'Cache-Control': 'max-age=86400' } def test__bp_static_url(self): @@ -213,8 +213,8 @@ class S3Tests(unittest.TestCase): exclude = ['/foo/static/foo.css', '/foo/static/foo/bar.css'] # we expect foo.css to be excluded and not uploaded expected = [call(bucket=None, name=u'/foo/static/bar.css'), + call().set_metadata('Cache-Control', 'cache instruction'), call().set_metadata('Expires', 'Thu, 31 Dec 2037 23:59:59 GMT'), - call().set_metadata('Cache-Control', 'max-age=86400'), call().set_metadata('Content-Encoding', 'gzip'), call().set_contents_from_filename('/home/z/bar.css')] flask_s3._write_files(self.app, static_url_loc, static_folder, assets, From 30190f77154b47278ea39a22ccb604676cdf8487 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Tue, 11 Jun 2013 01:38:03 +0200 Subject: [PATCH 3/4] Updates issue #7 - Set old S3_CACHE_CONTROL in S3_HEADERS on init and doc fixes. --- docs/index.rst | 12 +++--------- flask_s3.py | 7 +++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index a95768c..c6f49d9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -152,9 +152,9 @@ headers you want to use with the `S3_HEADERS` option. 'Cache-Control': 'max-age=86400', } -See `yahoo`_ more information on how to set good values for your headers. +See `Yahoo!`_ more information on how to set good values for your headers. -.. _yahoo: http://developer.yahoo.com/performance/rules.html#expires +.. _Yahoo!: http://developer.yahoo.com/performance/rules.html#expires .. _settings: .. _configuration: @@ -198,14 +198,8 @@ uploading assets to S3. specified by `flask.url_for`. `S3_HEADERS` Sets custom headers to be sent with each file to S3. **Default:** `{}` -`S3_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. - This sets the value of the Cache-Control header that - is set in the metadata when `S3_USE_CACHE_CONTRL` is - set to `True`. +`S3_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. `S3_USE_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. - Specifies whether or not to set the metadata for the - Cache-Control headers. - **Default:** `False` =========================== =================================================== .. _debug: http://flask.pocoo.org/docs/config/#configuration-basics diff --git a/flask_s3.py b/flask_s3.py index 043dbdb..5d41436 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -107,9 +107,6 @@ def _write_files(app, static_url_loc, static_folder, files, bucket, logger.debug("%s excluded from upload" % key_name) else: k = Key(bucket=bucket, name=key_name) - if (app.config['S3_USE_CACHE_CONTROL'] and - 'S3_CACHE_CONTROL' in app.config): - k.set_metadata('Cache-Control', app.config['S3_CACHE_CONTROL']) # Set custom headers for header, value in app.config['S3_HEADERS'].iteritems(): k.set_metadata(header, value) @@ -229,4 +226,6 @@ class FlaskS3(object): if app.config['USE_S3']: app.jinja_env.globals['url_for'] = url_for - + if 'S3_USE_CACHE_CONTROL' and 'S3_CACHE_CONTROL' in app.config: + cache_control_header = app.config['S3_CACHE_CONTROL'] + app.config['S3_HEADERS']['Cache-Control'] = cache_control_header From c07be84a57cbf9f60c63590bb7c243a69253e924 Mon Sep 17 00:00:00 2001 From: Hannes Ljungberg Date: Tue, 11 Jun 2013 01:41:11 +0200 Subject: [PATCH 4/4] Check for S3_USE_CACHE_CONTROL boolean --- flask_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flask_s3.py b/flask_s3.py index 5d41436..bbb8284 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -226,6 +226,6 @@ class FlaskS3(object): if app.config['USE_S3']: app.jinja_env.globals['url_for'] = url_for - if 'S3_USE_CACHE_CONTROL' and 'S3_CACHE_CONTROL' in app.config: + if app.config['S3_USE_CACHE_CONTROL'] and 'S3_CACHE_CONTROL' in app.config: cache_control_header = app.config['S3_CACHE_CONTROL'] app.config['S3_HEADERS']['Cache-Control'] = cache_control_header