From 0e374741d0d3fe18cefda532a6b248d72ef3458d Mon Sep 17 00:00:00 2001 From: Andrew Snowden Date: Mon, 26 Oct 2015 15:58:52 +0200 Subject: [PATCH 1/4] Allow GZIP compression of assets --- flask_s3.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index 67dc89f..92816de 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -3,6 +3,7 @@ import json import logging import os import re +import zlib from collections import defaultdict import boto3 @@ -175,6 +176,7 @@ def _static_folder_path(static_url, static_folder, static_asset): def _write_files(s3, app, static_url_loc, static_folder, files, bucket, ex_keys=None, hashes=None): """ Writes all the files inside a static folder to S3. """ + should_gzip = app.config.get('S3_GZIP') new_hashes = [] static_folder_rel = _path_to_relative_url(static_folder) for file_path in files: @@ -206,11 +208,19 @@ def _write_files(s3, app, static_url_loc, static_folder, files, bucket, for header, value in headers.iteritems(): h[header] = value + if should_gzip: + h["content-encoding"] = "gzip" + with open(file_path) as fp: metadata, params = split_metadata_params(merge_two_dicts(app.config['S3_HEADERS'], h)) + if should_gzip: + data = zlib.compress(fp.read()) + else: + data = fp.read() + s3.put_object(Bucket=bucket, Key=key_name, - Body=fp.read(), + Body=data, ACL="public-read", Metadata=metadata, **params) @@ -370,7 +380,8 @@ class FlaskS3(object): ('S3_HEADERS', {}), ('S3_FILEPATH_HEADERS', {}), ('S3_ONLY_MODIFIED', False), - ('S3_URL_STYLE', 'host')] + ('S3_URL_STYLE', 'host'), + ('S3_GZIP', False)] for k, v in defaults: app.config.setdefault(k, v) From 8714841deb3bc1e59594913f9d22bb80690d5380 Mon Sep 17 00:00:00 2001 From: Andrew Snowden Date: Mon, 26 Oct 2015 16:54:21 +0200 Subject: [PATCH 2/4] Mimetype and switch to GZIP --- flask_s3.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index 92816de..573cc9d 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -3,7 +3,9 @@ import json import logging import os import re -import zlib +import gzip +import cStringIO +import mimetypes from collections import defaultdict import boto3 @@ -210,11 +212,22 @@ def _write_files(s3, app, static_url_loc, static_folder, files, bucket, if should_gzip: h["content-encoding"] = "gzip" + if "content-type" not in h: + # When we use GZIP we have to explicitly set the content type + (mimetype, encoding) = mimetypes.guess_type(file_path, + False) + h["content-type"] = mimetype with open(file_path) as fp: metadata, params = split_metadata_params(merge_two_dicts(app.config['S3_HEADERS'], h)) if should_gzip: - data = zlib.compress(fp.read()) + compressed = cStringIO.StringIO() + z = gzip.GzipFile(os.path.basename(file_path), 'wb', 9, + compressed) + z.write(fp.read()) + z.close() + + data = compressed.getvalue() else: data = fp.read() From a7e8845d61424bf6927c8e9408f7d4a3ccebdae8 Mon Sep 17 00:00:00 2001 From: Andrew Snowden Date: Mon, 26 Oct 2015 16:58:13 +0200 Subject: [PATCH 3/4] Add readme line --- docs/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 5ed9621..6bd7458 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -229,6 +229,9 @@ uploading assets to S3. your S3 bucket to force all files to upload again. `S3_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. `S3_USE_CACHE_CONTROL` **Deprecated**. Please use `S3_HEADERS` instead. +`S3_GZIP` Compress all assets using GZIP and set the + corresponding Content-Type and Content-Encoding + headers on the S3 files. =========================== =================================================== .. _debug: http://flask.pocoo.org/docs/config/#configuration-basics From aa335301f3e7a7d92ead38140a39ca18eb2454be Mon Sep 17 00:00:00 2001 From: Andrew Snowden Date: Tue, 27 Oct 2015 10:15:48 +0200 Subject: [PATCH 4/4] Fix for Python 3 and warning for unknown mimetype --- flask_s3.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/flask_s3.py b/flask_s3.py index 573cc9d..a4e80b9 100644 --- a/flask_s3.py +++ b/flask_s3.py @@ -4,7 +4,10 @@ import logging import os import re import gzip -import cStringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO import mimetypes from collections import defaultdict @@ -216,12 +219,16 @@ def _write_files(s3, app, static_url_loc, static_folder, files, bucket, # When we use GZIP we have to explicitly set the content type (mimetype, encoding) = mimetypes.guess_type(file_path, False) - h["content-type"] = mimetype + if mimetype: + h["content-type"] = mimetype + else: + logger.warn("Unable to detect mimetype for %s" % + file_path) with open(file_path) as fp: metadata, params = split_metadata_params(merge_two_dicts(app.config['S3_HEADERS'], h)) if should_gzip: - compressed = cStringIO.StringIO() + compressed = StringIO() z = gzip.GzipFile(os.path.basename(file_path), 'wb', 9, compressed) z.write(fp.read())