commit a8e9c60dacaeda899c2e5e3ea4aa5b93fa2d548f Author: Mardix Date: Thu May 21 01:17:09 2015 -0400 0.1.0 - First commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95e8d14 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +*.py[co] +__pycache__* + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +MANIFEST + +# PyCharm +.idea/* +.idea/libraries/sass_stdlib.xml + +# Distribution +dist/* + +# Mac stuff +.DS_Store + +docs/_build* + +.test_config + +.tox \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..d45d49b --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,3 @@ + +0.1.0 + - First \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000..11e8d27 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Mardix + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c2ec27a --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# Flask-CloudStorage + +A simple Flask Extension (also standalone) library to upload and save files on the cloud. + +Supported storage: + +- AWS S3 +- Google Storage +- Microsoft Azure +- Rackspace CloudFiles +- Local (for local file system) + +## Install + + pip install flask-cloudstorage + +--- + +(c) 2015 Mardix \ No newline at end of file diff --git a/flask_cloudstorage.py b/flask_cloudstorage.py new file mode 100644 index 0000000..186652a --- /dev/null +++ b/flask_cloudstorage.py @@ -0,0 +1,273 @@ +""" +Flask-CloudStorage +""" + +import os +from werkzeug.utils import secure_filename +from werkzeug.datastructures import FileStorage +from importlib import import_module +import shortuuid +from libcloud.storage.types import Provider, ObjectDoesNotExistError +from libcloud.storage.providers import get_driver +from libcloud.storage.base import Object +from libcloud.storage.drivers.local import LocalStorageDriver + + +# Extension +EXTENSIONS = { + "TEXT": ["txt"], + "DOCUMENT": ["rtf", "odf", "ods", "gnumeric", "abw", "doc", "docx", "xls", "xlsx"], + "IMAGE": ["jpg", "jpeg", "jpe", "png", "gif", "svg", "bmp"], + "AUDIO": ["wav", "mp3", "aac", "ogg", "oga", "flac"], + "DATA": ["csv", "ini", "json", "plist", "xml", "yaml", "yml"], + "SCRIPTS": ["js", "php", "pl", "py", "rb", "sh"], + "ARCHIVES": ["gz", "bz2", "zip", "tar", "tgz", "txz", "7z"] +} + +ALL_EXTENSIONS = EXTENSIONS["TEXT"] \ + + EXTENSIONS["DOCUMENT"] \ + + EXTENSIONS["IMAGE"] \ + + EXTENSIONS["AUDIO"] \ + + EXTENSIONS["DATA"] + +def get_filename(filename): + return os.path.splitext(filename)[0] + +def get_file_extension(filename): + """ + Return the file extension without the dot + :param filename: + :return: + """ + return os.path.splitext(filename)[1][1:].lower() + +def get_file_extension_type(filename): + ext = get_file_extension(filename) + if ext: + for name, group in EXTENSIONS.items(): + if ext in group: + return name + return "OTHER" + +class InvalidExtensionError(Exception): + pass + +class Storage(object): + _container_name = None + _container = None + _driver = None + allowed_extensions = EXTENSIONS["TEXT"] \ + + EXTENSIONS["DOCUMENT"] \ + + EXTENSIONS["IMAGE"] \ + + EXTENSIONS["AUDIO"] \ + + EXTENSIONS["DATA"] + + def __init__(self, provider=None, + key=None, + secret=None, + container=None, + local_path=None, + allowed_extensions=None, + app=None, + **kwargs): + if app: + self.init_app(app) + + if allowed_extensions: + self.allowed_extensions = allowed_extensions + + if provider: + if not key and local_path: + key = local_path + + kwparams = { + "key": key, + "secret": secret + } + + kwparams.update(kwargs) + self.driver = self.get_driver_class(provider)(**kwparams) + + if container: + self.container = container + + def init_app(self, app): + """ + To initiate with Flask + :param app: + :return: + """ + provider = app.config.get("CLOUDSTORAGE_PROVIDER", None) + key = app.config.get("CLOUDSTORAGE_KEY", None) + secret = app.config.get("CLOUDSTORAGE_SECRET", None) + container = app.config.get("CLOUDSTORAGE_CONTAINER", None) + local_path = app.config.get("CLOUDSTORAGE_LOCAL_PATH", None) + allowed_extensions = app.config.get("CLOUDSTORAGE_ALLOWED_EXTENSIONS", None) + + if provider.upper() == "LOCAL": + if not local_path: + raise ValueError("For 'LOCAL' provider, Storage requires CLOUDSTORAGE_LOCAL_PATH") + else: + key = local_path + secret = None + + self.__init__(provider=provider, + key=key, + secret=secret, + container=container, + local_path=local_path, + allowed_extensions=allowed_extensions) + + @property + def driver(self): + return self._driver + + @driver.setter + def driver(self, driver): + self._driver = driver + + @property + def container(self): + return self._container + + @container.setter + def container(self, container_name): + self._container = self.driver.get_container(container_name) + + def __iter__(self): + """ + Iterate over all the files in the container + :return: generator + """ + for obj in self.container.iterate_objects(): + yield Object(obj, cloudstorage=self) + + @classmethod + def get_driver_class(cls, provider): + if "." not in provider: + driver = getattr(Provider, provider.upper()) + else: + parts = provider.split('.') + kls = parts.pop() + path = '.'.join(parts) + module = import_module(path) + if not hasattr(module, kls): + raise ImportError('{0} provider not found at {1}'.format( + kls, + path)) + driver = getattr(module, kls) + return get_driver(driver) + + def object(self, object_name, size=0, hash=None, extra=None, meta_data={}): + obj = Object(name=object_name, + size=size, + hash=hash, + extra=extra, + meta_data=meta_data, + container=self.container, + driver=self.driver) + return StorageObject(obj, cloudstorage=self) + + def upload(self, file, object_name, + acl="private", + meta_data={}, + allowed_extensions=None, + overwrite=False): + + extra = { + "meta_data": meta_data, + "acl": acl + } + object_name = object_name.strip("/").strip() + + if isinstance(self.driver, LocalStorageDriver): + object_name = secure_filename(object_name) + + if isinstance(file, FileStorage): + extension = get_file_extension(file.filename) + else: + extension = get_file_extension(file) + + if not allowed_extensions: + allowed_extensions = self.allowed_extensions + if extension.lower() not in allowed_extensions: + raise InvalidExtensionError("Invalid file extension") + + if not overwrite: + object_name = self._safe_object_name(object_name) + + obj = self.container.upload_object(file_path=file, + object_name=object_name, + extra=extra) + return StorageObject(obj=obj, cloudstorage=self) + + def object_exists(self, object_name): + """ + Test if object exists + :param object_name: + :return bool: + """ + try: + container_name = self.container.name + self.driver.get_object(container_name, object_name) + return True + except ObjectDoesNotExistError: + return False + + def _safe_object_name(self, object_name): + """ If the file already exists the file will be renamed to contain a + short url safe UUID. This will avoid overwtites. + Arguments + --------- + filename : str + A filename to check if it exists + Returns + ------- + str + A safe filenaem to use when writting the file + """ + filename = get_filename(object_name) + extension = get_file_extension(object_name) + while self.object_exists(object_name): + uuid = shortuuid.uuid() + object_name = "%s_%s.%s" % (filename, uuid, extension) + return object_name + +class StorageObject(object): + """ + attr: + name + size + hash + container + extra + meta_data + driver + + download + delete + """ + def __init__(self, obj, cloudstorage=None): + self.obj = obj + self.cloudstorage = cloudstorage + + def __getattr__(self, item): + return getattr(self.obj, item) + + def __len__(self): + return self.size + + @property + def url(self): + return self.obj.get_cdn_url() + + @property + def extension(self): + return get_file_extension(self.name) + + @property + def type(self): + return get_file_extension_type(self.name) + + def exists(self): + return self.cloudstorage.object_exists(self.name) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dcc50a1 --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +""" +Flask-CloudStorage + +A simple Flask Extension (also standalone) library to upload and save files on the cloud. + +Supported storage: + +- AWS S3 +- Google Storage +- Microsoft Azure +- Rackspace CloudFiles +- Local + +""" + +from setuptools import setup, find_packages + + +__NAME__ = "flask-CloudStorage" +__version__ = "0.1.0" +__author__ = "Mardix" +__license__ = "MIT" +__copyright__ = "2015" + +setup( + name=__NAME__, + version=__version__, + license=__license__, + author=__author__, + author_email='mardix@github.com', + description="Flask-CloudStorage is a simple flask extension and standalone library to upload and save files on the cloud", + long_description=__doc__, + url='https://github.com/mardix/flask-cloudstorage/', + download_url='http://github.com/mardix/flask-cloudstorage/tarball/master', + py_modules=['flask_cloudstorage'], + include_package_data=True, + packages=find_packages(), + install_requires=[ + "apache-libcloud==0.17.0", + "lockfile==0.10.2", + "shortuuid==0.1" + ], + + keywords=["flask", "s3", "aws", "cloudfiles", "storage", "azure", "google"], + platforms='any', + classifiers=[ + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Software Development :: Libraries :: Python Modules' + ], + zip_safe=False +) + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..4e5980e --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +__author__ = 'mardochee.macxis' diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 0000000..c09f9bd --- /dev/null +++ b/tests/config.py @@ -0,0 +1,14 @@ + +# ON S3 +PROVIDER = "S3" +KEY = "" +SECRET = "" +CONTAINER = "" +CONTAINER_2 = "" + + +# FOR LOCAL +PROVIDER = "LOCAL" +CONTAINER = "container_1" +CONTAINER_2 = "container_2" +LOCAL_PATH = "" \ No newline at end of file diff --git a/tests/container_1/empty b/tests/container_1/empty new file mode 100644 index 0000000..e69de29 diff --git a/tests/container_2/empty b/tests/container_2/empty new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/hello,js b/tests/data/hello,js new file mode 100644 index 0000000..575a91b --- /dev/null +++ b/tests/data/hello,js @@ -0,0 +1 @@ +// This is the javascript file diff --git a/tests/data/hello.txt b/tests/data/hello.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cloudstorage.py b/tests/test_cloudstorage.py new file mode 100644 index 0000000..72a663f --- /dev/null +++ b/tests/test_cloudstorage.py @@ -0,0 +1,115 @@ +import os + +import pytest +from libcloud.storage.base import (StorageDriver, + Container) +from tests import config +from flask_cloudstorage import (get_file_extension, + get_file_extension_type, + Storage, + StorageObject, + InvalidExtensionError) + +CWD = os.path.dirname(__file__) + +# Manipulate +class App(object): + config = config=dict( + CLOUDSTORAGE_PROVIDER=config.PROVIDER, + CLOUDSTORAGE_KEY=config.KEY, + CLOUDSTORAGE_SECRET=config.SECRET, + CLOUDSTORAGE_CONTAINER=config.CONTAINER, + CLOUDSTORAGE_LOCAL_PATH=CWD, + CLOUDSTORAGE_ALLOWED_EXTENSIONS=[]) + +def _setup_function(): + pass + +def _teardown_function(): + pass + +def test_get_file_extension(): + filename = "hello.jpg" + assert get_file_extension(filename) == "jpg" + +def test_get_file_extension_type(): + filename = "hello.mp3" + assert get_file_extension_type(filename) == "AUDIO" + +#--- + +app = App() + +def app_storage(): + return Storage(app=app) + +def test_get_driver_class(): + driver = Storage.get_driver_class("S3") + assert isinstance(driver, type) + +def test_driver(): + storage = app_storage() + assert isinstance(storage.driver, StorageDriver) + +def test_container(): + storage = app_storage() + assert isinstance(storage.container, Container) + +def test_set_container(): + storage = app_storage() + storage.container = config.CONTAINER_2 + assert storage.container.name == config.CONTAINER_2 + +def test_flask_app(): + storage = app_storage() + assert isinstance(storage.driver, StorageDriver) + +def _test_iter(): + storage = app_storage() + l = [o for o in storage] + assert isinstance(l, list) + +def test_storage_object_not_exists(): + object_name = "hello.png" + storage = app_storage() + assert storage.object_exists(object_name) is False + +def test_storage_object(): + object_name = "hello.txt" + storage = app_storage() + o = storage.object(object_name) + assert isinstance(o, StorageObject) + +def test_object_type_extension(): + object_name = "hello.jpg" + storage = app_storage() + o = storage.object(object_name) + assert o.type == "IMAGE" + assert o.extension == "jpg" + +def test_object_not_exists(): + object_name = "hello.png" + storage = app_storage() + o = storage.object(object_name) + assert o.exists() is False + +def test_storage_upload_invalid(): + storage = app_storage() + object_name = "my-js/hello.js" + with pytest.raises(InvalidExtensionError): + storage.upload(CWD + "/data/hello.js", object_name) + +def test_storage_upload_ovewrite(): + storage = app_storage() + object_name = "my-txt-hello.txt" + o = storage.upload(CWD + "/data/hello.txt", object_name, overwrite=True) + assert isinstance(o, StorageObject) + assert o.name == object_name + +def test_storage_upload(): + storage = app_storage() + object_name = "my-txt-hello2.txt" + storage.upload(CWD + "/data/hello.txt", object_name) + o = storage.upload(CWD + "/data/hello.txt", object_name) + assert isinstance(o, StorageObject) + assert o.name != object_name \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..b3cd340 --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +envlist = py26,py27 +[testenv] +deps=pytest +commands=py.test \ No newline at end of file