mirror of
https://github.com/wassname/flask-cloudy.git
synced 2026-08-20 12:20:28 +08:00
0.7
This commit is contained in:
Mardix
@@ -1,3 +1,6 @@
|
||||
0.7.0
|
||||
- Removed LOCAL_PATH configuration. Use CONTAINER as the LOCAL_PATH
|
||||
|
||||
0.6.0
|
||||
- More pythonic
|
||||
- implement __contains__ to look for an item in the storage. `if object_name in storage`
|
||||
|
||||
@@ -1,43 +1,486 @@
|
||||
# Flask-CloudStorage
|
||||
|
||||
A wrapper around Apache-Libcloud to upload and save files on cloud storage
|
||||
providers such as: AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles,
|
||||
and even on local storage through a Flask application.
|
||||
(It can be used as standalone)
|
||||
|
||||
For local file storage, it provides a flask endpoint to access the files
|
||||
|
||||
Supported storage:
|
||||
|
||||
- AWS S3
|
||||
- Google Storage
|
||||
- Microsoft Azure
|
||||
- Rackspace CloudFiles
|
||||
- Local (for local file system)
|
||||
|
||||
## Install
|
||||
|
||||
pip install flask-cloudstorage
|
||||
# Flask-Cloudy
|
||||
|
||||
|
||||
### Example of uploading a file
|
||||
## About
|
||||
|
||||
from flask import Flask, request
|
||||
from flask_cloudstorage import Storage
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
storage = Storage(app=app)
|
||||
|
||||
@route("/upload", methods=["POST", "GET"]):
|
||||
def upload():
|
||||
if request.method == "POST":
|
||||
my_upload = storage.upload(request.file.get("file"))
|
||||
name = my_upload.name
|
||||
size = my_upload.size
|
||||
url = my_upload.get_url()
|
||||
return url
|
||||
|
||||
A Flask extension to **access, upload, download, save and delete** files on cloud storage providers such as:
|
||||
AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles, and even Local file system.
|
||||
|
||||
For local file storage, it also provides a flask endpoint to access the files.
|
||||
|
||||
---
|
||||
|
||||
(c) 2015 Mardix
|
||||
##TLDR; Quick Example
|
||||
|
||||
from flask import Flask, request
|
||||
from flask_cloudy import Storage
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Update the config
|
||||
app.config.update({
|
||||
"STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc...
|
||||
"STORAGE_KEY": "",
|
||||
"STORAGE_SECRET": "",
|
||||
"STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud
|
||||
"STORAGE_SERVER": True,
|
||||
"STORAGE_SERVER_URL": "/files" # The url endpoint to access files on LOCAL provider
|
||||
})
|
||||
|
||||
# Setup storage
|
||||
storage = new Storage()
|
||||
storage.init_app(app)
|
||||
|
||||
@app.route("/upload", methods=["POST", "GET"]):
|
||||
def upload():
|
||||
if request.method == "POST":
|
||||
file = request.file.get("file")
|
||||
my_upload = storage.upload(file)
|
||||
|
||||
# some useful properties
|
||||
name = my_upload.name
|
||||
extension = my_upload.extension
|
||||
size = my_upload.size
|
||||
url = my_upload.url
|
||||
|
||||
return url
|
||||
|
||||
# Pretending the file uploaded is "my-picture.jpg"
|
||||
# it will return a url in the format: http://domain.com/files/my-picture.jpg
|
||||
|
||||
|
||||
# A download endpoint, to download the file
|
||||
@app.route("/download/<path:object_name>"):
|
||||
def download(object_name):
|
||||
my_object = storage.get(object_name)
|
||||
if my_object:
|
||||
download_url = my_object.download()
|
||||
return download_url
|
||||
else:
|
||||
abort(404, "File doesn't exist")
|
||||
|
||||
---
|
||||
|
||||
|
||||
### Features:
|
||||
|
||||
- Browse files
|
||||
|
||||
- Upload files
|
||||
|
||||
- Download files
|
||||
|
||||
- Delete files
|
||||
|
||||
- Serve files via http
|
||||
|
||||
|
||||
### Supported storage:
|
||||
|
||||
- AWS S3
|
||||
|
||||
- Google Storage
|
||||
|
||||
- Microsoft Azure
|
||||
|
||||
- Rackspace CloudFiles
|
||||
|
||||
- Local (for local file system)
|
||||
|
||||
|
||||
**Dependecies:** (They will be installed upon setup)
|
||||
|
||||
- Flask
|
||||
|
||||
- Apache-Libcloud
|
||||
|
||||
---
|
||||
|
||||
## Install & Config
|
||||
|
||||
pip install flask-cloudy
|
||||
|
||||
---
|
||||
|
||||
(To use it as standalone, refer to API documentaion below)
|
||||
|
||||
## Config for Flask
|
||||
|
||||
Within your Flask application's settings you can provide the following settings to control
|
||||
the behavior of Flask-Cloudy
|
||||
|
||||
|
||||
**- STORAGE_PROVIDER** (str)
|
||||
|
||||
- LOCAL
|
||||
- S3
|
||||
- S3_US_WEST
|
||||
- S3_US_WEST_OREGON
|
||||
- S3_EU_WEST
|
||||
- S3_AP_SOUTHEAST
|
||||
- S3_AP_NORTHEAST
|
||||
- GOOGLE_STORAGE
|
||||
- AZURE_BLOBS
|
||||
- CLOUDFILES
|
||||
|
||||
|
||||
**- STORAGE_KEY** (str)
|
||||
|
||||
The access key of the cloud storage provider
|
||||
|
||||
None for LOCAL
|
||||
|
||||
**- STORAGE_SECRET** (str)
|
||||
|
||||
The access secret key of the cloud storage provider
|
||||
|
||||
None for LOCAL
|
||||
|
||||
**- STORAGE_CONTAINER** (str)
|
||||
|
||||
The *BUCKET NAME* for cloud storage providers
|
||||
|
||||
For *LOCAL* provider, this is the local directory path
|
||||
|
||||
|
||||
**STORAGE_ALLOWED_EXTENSIONS** (list)
|
||||
|
||||
List of all extensions to allow
|
||||
|
||||
Example: ["png", "jpg", "jpeg", "mp3"]
|
||||
|
||||
**STORAGE_SERVER** (bool)
|
||||
|
||||
For *LOCAL* provider only.
|
||||
|
||||
True to expose the files in the container so they can be accessed
|
||||
|
||||
Default: *True*
|
||||
|
||||
**STORAGE_SERVER_URL** (str)
|
||||
|
||||
For *LOCAL* provider only.
|
||||
|
||||
The endpoint to access the files from the local storage.
|
||||
|
||||
Default: */files*
|
||||
|
||||
---
|
||||
|
||||
## API Documention
|
||||
|
||||
Flask-Cloudy is a wrapper around Apache-Libcloud, the Storage class gives you access to Driver and Container of Apache-Libcloud.
|
||||
|
||||
*Lexicon:*
|
||||
|
||||
Object: A file or a file path.
|
||||
|
||||
Container: The main directory, or a bucket name containing all the objects
|
||||
|
||||
Provider: The method
|
||||
|
||||
Storage:
|
||||
|
||||
### flask_cloudy.Storage
|
||||
|
||||
The **Storage** class allows you to access, upload, get an object from the Storage.
|
||||
|
||||
##### Storage(provider, key=None, secret=None, container=None)
|
||||
|
||||
- provider: the storage provider:
|
||||
|
||||
- LOCAL
|
||||
- S3
|
||||
- S3_US_WEST
|
||||
- S3_US_WEST_OREGON
|
||||
- S3_EU_WEST
|
||||
- S3_AP_SOUTHEAST
|
||||
- S3_AP_NORTHEAST
|
||||
- GOOGLE_STORAGE
|
||||
- AZURE_BLOBS
|
||||
- CLOUDFILES
|
||||
|
||||
- key: The access key of the cloud storage. None when provider is LOCAL
|
||||
|
||||
- secret: The secret access key of the cloud storage. None when provider is LOCAL
|
||||
|
||||
- container:
|
||||
|
||||
- For cloud storage, use the **BUCKET NAME**
|
||||
|
||||
- For LOCAL provider, it's the directory path where to access the files
|
||||
|
||||
|
||||
##### Storage.init_app(app)
|
||||
|
||||
To initiate the Storage via Flask config.
|
||||
|
||||
It will also setup a server endpoint when STORAGE_PROVIDER == LOCAL
|
||||
|
||||
from flask import Flask, request
|
||||
from flask_cloudy import Storage
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Update the config
|
||||
app.config.update({
|
||||
"STORAGE_PROVIDER": "LOCAL", # Can also be S3, GOOGLE_STORAGE, etc...
|
||||
"STORAGE_KEY": "",
|
||||
"STORAGE_SECRET": "",
|
||||
"STORAGE_CONTAINER": "./", # a directory path for local, bucket name of cloud
|
||||
"STORAGE_SERVER": True,
|
||||
"STORAGE_SERVER_URL": "/files"
|
||||
})
|
||||
|
||||
# Setup storage
|
||||
storage = new Storage()
|
||||
storage.init_app(app)
|
||||
|
||||
@app.route("/upload", methods=["POST", "GET"]):
|
||||
def upload():
|
||||
if request.method == "POST":
|
||||
file = request.file.get("file")
|
||||
my_upload = storage.upload(file)
|
||||
|
||||
# some useful properties
|
||||
name = my_upload.name
|
||||
extension = my_upload.extension
|
||||
size = my_upload.size
|
||||
url = my_upload.url
|
||||
|
||||
return url
|
||||
|
||||
# Pretending the file uploaded is "my-picture.jpg"
|
||||
# it will return a url in the format: http://domain.com/files/my-picture.jpg
|
||||
|
||||
|
||||
|
||||
##### Storage.get(object_name)
|
||||
|
||||
Get an object in the storage by name, relative to the container.
|
||||
|
||||
It will return an instance of **flask_cloudy.Object**
|
||||
|
||||
- object_name: The name of the object.
|
||||
|
||||
Some valid object names, they can contains slashes to indicate it's a directory
|
||||
|
||||
|
||||
- file.txt
|
||||
|
||||
- my_dir/file.txt
|
||||
|
||||
- my_dir/sub_dir/file.txt
|
||||
|
||||
.
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
object_name = "hello.txt"
|
||||
my_object = storage.get(object_name)
|
||||
|
||||
|
||||
|
||||
##### Storage.upload(file, name=None, prefix=None, allowed_extesion=[], overwrite=Flase, public=False)
|
||||
|
||||
To save or upload a file in the container
|
||||
|
||||
- file: the string of the file location or a file object
|
||||
|
||||
- name: to give the file a new name
|
||||
|
||||
- prefix: a name to add in front of the file name. It can make it a directory
|
||||
|
||||
- allowed_extensions: list of extensions
|
||||
|
||||
- overwrite: If True it will overwrite existing files, otherwise it will add a uuid in the file name to make it unique
|
||||
|
||||
- public: Bool - To set the **acl** to *public-read* when True, *private* when False
|
||||
|
||||
.
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
my_file = "my_dir/readme.md"
|
||||
|
||||
|
||||
**1) This example will upload the file, an assign the object the name of the file**
|
||||
|
||||
storage.upload(my_file)
|
||||
|
||||
|
||||
**2) This example will upload the file, an assign the object the name of the file**
|
||||
|
||||
storage.upload(my_file, name="new_readme")
|
||||
|
||||
The uploaded file will be named: **new_readme.md**
|
||||
|
||||
|
||||
**3) Put the uploaded file under a different location**
|
||||
|
||||
storage.upload(my_file, name="new_readme", prefix="my_dir/")
|
||||
|
||||
now the filename becomes **my_dir/new_readme.md**
|
||||
|
||||
On LOCAL it will create the directory *my_dir* if it doesn't exist.
|
||||
|
||||
|
||||
**4a.) Public upload **
|
||||
|
||||
storage.upload(my_file, public=True)
|
||||
|
||||
**4b.) Private upload **
|
||||
|
||||
storage.upload(my_file, public=False)
|
||||
|
||||
|
||||
##### Storage.create(object_name, size=0, hash=None, extra=None, metda_data=None)
|
||||
|
||||
Explicitly create an object that may exist already. Usually, when paramameters (name, size, hash, etc...) are already saved, let's say in the database, and you want Storage to manipulate the file.
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
existing_name = "holla.txt"
|
||||
existing_size = "8000" # in bytes
|
||||
new_object = storage.create(object_name=existing_name, size=existing_size)
|
||||
|
||||
# Now I can do
|
||||
url = new_object.url
|
||||
size = len(new_object)
|
||||
|
||||
*It's Pythonic!!!*
|
||||
|
||||
##### Iterate through all the objects in the container
|
||||
|
||||
Each object is an instance on **flask_cloudy.Object**
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
for obj in storage:
|
||||
print(obj.name)
|
||||
|
||||
##### Get the total objects in the container
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
total_items = len(storage)
|
||||
|
||||
##### Check to see if an object exists in the container
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
my_file = "hello.txt"
|
||||
|
||||
if my_file in storage:
|
||||
print("File is in the storage")
|
||||
|
||||
---
|
||||
|
||||
|
||||
### flask_cloudy.Object
|
||||
|
||||
The class **Object** is an entity of an object in the container.
|
||||
|
||||
Usually, you will get a cloud object by accessing an object in the container.
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
my_object = storage.get("my_object.txt")
|
||||
|
||||
Properties:
|
||||
|
||||
##### Object.name
|
||||
|
||||
The name of the object
|
||||
|
||||
|
||||
##### Object.size
|
||||
|
||||
The size in bytes of the object
|
||||
|
||||
|
||||
##### Object.extension
|
||||
|
||||
The extension of the object
|
||||
|
||||
|
||||
##### Object.url
|
||||
|
||||
Get the full url of the object
|
||||
|
||||
|
||||
##### Object.short_url
|
||||
|
||||
Specially for LOCAL provider, it will return the url without the domain.
|
||||
|
||||
For cloud providers, it will return the full url just like **Object.url**
|
||||
|
||||
|
||||
##### Object.secure_url
|
||||
|
||||
Return a secured url, with **https://**
|
||||
|
||||
##### Object.path
|
||||
|
||||
The path of the object relative to the container
|
||||
|
||||
|
||||
##### Object.provider_name
|
||||
|
||||
The provider name: ie: Local, S3,...
|
||||
|
||||
|
||||
##### Object.type
|
||||
|
||||
The type of the object, ie: IMAGE, AUDIO, TEXT,... OTHER
|
||||
|
||||
|
||||
Methods:
|
||||
|
||||
##### Object.save_to(destination, name=None, overwrite=False, delete_on_failure=True)
|
||||
|
||||
To save the object to a local path
|
||||
|
||||
- destination: The directory to save the object to
|
||||
|
||||
- name: To rename the file in the local directory. Do not put the extension of the file, it will append automatically
|
||||
|
||||
- overwrite: bool - To overwrite the file if it exists
|
||||
|
||||
- delete_on_failure: bool - To delete the file it fails to save
|
||||
|
||||
.
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
my_object = storage.get("my_object.txt")
|
||||
my_new_path = "/my/new/path"
|
||||
my_new_file = my_object.save_to(my_new_path)
|
||||
|
||||
print(my_new_file) # Will print -> /my/new/path/my_object.txt
|
||||
|
||||
|
||||
##### Object.download(name=None)
|
||||
|
||||
Return a URL that triggers the browser download of the file.
|
||||
|
||||
Use the url to download the file
|
||||
|
||||
storage = Storage(provider, key, secret, container)
|
||||
my_object = storage.get("my_object.txt")
|
||||
download_url = my_object.download()
|
||||
|
||||
# or with flask
|
||||
|
||||
@app.route("/download/<path:object_name>"):
|
||||
def download(object_name):
|
||||
my_object = storage.get(object_name)
|
||||
if my_object:
|
||||
download_url = my_object.download()
|
||||
return download_url
|
||||
else:
|
||||
abort(404, "File doesn't exist")
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
Thank you
|
||||
|
||||
Mardix :)
|
||||
|
||||
---
|
||||
|
||||
License: MIT - Copyright 2015 Mardix
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Flask-CloudStorage
|
||||
Flask-Cloudy
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -7,7 +7,7 @@ import warnings
|
||||
from werkzeug.utils import secure_filename
|
||||
from werkzeug.datastructures import FileStorage
|
||||
from importlib import import_module
|
||||
from flask import send_file, abort, url_for
|
||||
from flask import send_file, abort, url_for, request
|
||||
import shortuuid
|
||||
from libcloud.storage.types import Provider, ObjectDoesNotExistError
|
||||
from libcloud.storage.providers import DRIVERS, get_driver
|
||||
@@ -16,12 +16,12 @@ from libcloud.storage.drivers import local
|
||||
from six.moves.urllib.parse import urlparse, urlunparse, urljoin
|
||||
import slugify
|
||||
|
||||
FILE_SERVER_ENDPOINT = "FLASK_CLOUDSTORAGE:FILE_SERVER"
|
||||
SERVER_ENDPOINT = "FLASK_CLOUDY_SERVER"
|
||||
|
||||
EXTENSIONS = {
|
||||
"TEXT": ["txt"],
|
||||
"TEXT": ["txt", "md"],
|
||||
"DOCUMENT": ["rtf", "odf", "ods", "gnumeric", "abw", "doc", "docx", "xls", "xlsx"],
|
||||
"IMAGE": ["jpg", "jpeg", "jpe", "png", "gif", "svg", "bmp"],
|
||||
"IMAGE": ["jpg", "jpeg", "jpe", "png", "gif", "svg", "bmp", "webp"],
|
||||
"AUDIO": ["wav", "mp3", "aac", "ogg", "oga", "flac"],
|
||||
"DATA": ["csv", "ini", "json", "plist", "xml", "yaml", "yml"],
|
||||
"SCRIPT": ["js", "php", "pl", "py", "rb", "sh"],
|
||||
@@ -35,6 +35,9 @@ ALL_EXTENSIONS = EXTENSIONS["TEXT"] \
|
||||
+ EXTENSIONS["DATA"] \
|
||||
+ EXTENSIONS["ARCHIVE"]
|
||||
|
||||
class InvalidExtensionError(Exception):
|
||||
pass
|
||||
|
||||
def get_file_name(filename):
|
||||
"""
|
||||
Return the filename without the path
|
||||
@@ -96,16 +99,10 @@ def get_provider_name(driver):
|
||||
return d
|
||||
return None
|
||||
|
||||
class InvalidExtensionError(Exception):
|
||||
pass
|
||||
|
||||
class LocalPathUndefinedError(Exception):
|
||||
pass
|
||||
|
||||
class Storage(object):
|
||||
_container_name = None
|
||||
_container = None
|
||||
_driver = None
|
||||
container = None
|
||||
driver = None
|
||||
config = {}
|
||||
allowed_extensions = EXTENSIONS["TEXT"] \
|
||||
+ EXTENSIONS["DOCUMENT"] \
|
||||
@@ -113,14 +110,13 @@ class Storage(object):
|
||||
+ EXTENSIONS["AUDIO"] \
|
||||
+ EXTENSIONS["DATA"]
|
||||
|
||||
def __init__(self, provider=None,
|
||||
def __init__(self,
|
||||
provider=None,
|
||||
key=None,
|
||||
secret=None,
|
||||
container=None,
|
||||
local_path=None,
|
||||
allowed_extensions=None,
|
||||
app=None,
|
||||
secure_url=False,
|
||||
**kwargs):
|
||||
|
||||
"""
|
||||
@@ -129,14 +125,11 @@ class Storage(object):
|
||||
:param key: str - provider key
|
||||
:param secret: str - provider secret
|
||||
:param container: str - the name of the container (bucket or a dir name if local)
|
||||
:param local_path: str - when provider == LOCAL, it is the base directory
|
||||
:param allowed_extensions: list - extensions allowed for upload
|
||||
:param app: Flask object -
|
||||
:param secure_url: bool - when getting the url, it will add https if true
|
||||
:param kwargs: any other params will pass to the provider initialization
|
||||
:return:
|
||||
"""
|
||||
self.secure_url = secure_url
|
||||
|
||||
if app:
|
||||
self.init_app(app)
|
||||
@@ -145,21 +138,22 @@ class Storage(object):
|
||||
self.allowed_extensions = allowed_extensions
|
||||
|
||||
if provider:
|
||||
if not key and local_path:
|
||||
key = local_path
|
||||
|
||||
kwparams = {
|
||||
"key": key,
|
||||
"secret": secret
|
||||
}
|
||||
|
||||
if "local" in provider.lower():
|
||||
kwparams["key"] = container
|
||||
container = ""
|
||||
|
||||
kwparams.update(kwargs)
|
||||
|
||||
self.driver = get_driver_class(provider)(**kwparams)
|
||||
if not isinstance(self.driver, StorageDriver):
|
||||
raise AttributeError("Invalid Driver")
|
||||
|
||||
if container:
|
||||
self.container = container
|
||||
|
||||
self.local_path = local_path
|
||||
self.container = self.driver.get_container(container)
|
||||
|
||||
def __iter__(self):
|
||||
"""
|
||||
@@ -168,9 +162,7 @@ class Storage(object):
|
||||
:return: generator
|
||||
"""
|
||||
for obj in self.container.iterate_objects():
|
||||
yield Object(obj=obj,
|
||||
secure_url=self.secure_url,
|
||||
local_path=self.local_path)
|
||||
yield Object(obj=obj)
|
||||
|
||||
def __len__(self):
|
||||
"""
|
||||
@@ -188,86 +180,59 @@ class Storage(object):
|
||||
:return bool:
|
||||
"""
|
||||
try:
|
||||
container_name = self.container.name
|
||||
self.driver.get_object(container_name, object_name)
|
||||
self.driver.get_object(self.container.name, object_name)
|
||||
return True
|
||||
except ObjectDoesNotExistError:
|
||||
return False
|
||||
|
||||
@property
|
||||
def driver(self):
|
||||
return self._driver
|
||||
|
||||
@driver.setter
|
||||
def driver(self, driver):
|
||||
if not isinstance(driver, StorageDriver):
|
||||
raise AttributeError("Invalid Driver")
|
||||
self._driver = driver
|
||||
|
||||
@property
|
||||
def container(self):
|
||||
return self._container
|
||||
|
||||
@container.setter
|
||||
def container(self, container_name):
|
||||
self._container = self.driver.get_container(container_name)
|
||||
|
||||
def init_app(self, app):
|
||||
"""
|
||||
To initiate with Flask
|
||||
:param app: Flask object
|
||||
:return:
|
||||
"""
|
||||
provider = app.config.get("CLOUDSTORAGE_PROVIDER", None)
|
||||
key = app.config.get("CLOUDSTORAGE_KEY", None)
|
||||
secret = app.config.get("CLOUDSTORAGE_SECRET", None)
|
||||
container = app.config.get("CLOUDSTORAGE_CONTAINER", None)
|
||||
local_path = app.config.get("CLOUDSTORAGE_LOCAL_PATH", None)
|
||||
allowed_extensions = app.config.get("CLOUDSTORAGE_ALLOWED_EXTENSIONS", None)
|
||||
secure_url = app.config.get("CLOUDSTORAGE_SERVE_FILES_URL_SECURE", False)
|
||||
serve_files = app.config.get("CLOUDSTORAGE_SERVE_FILES", False)
|
||||
serve_files_url = app.config.get("CLOUDSTORAGE_SERVE_FILES_URL", "files")
|
||||
provider = app.config.get("STORAGE_PROVIDER", None)
|
||||
key = app.config.get("STORAGE_KEY", None)
|
||||
secret = app.config.get("STORAGE_SECRET", None)
|
||||
container = app.config.get("STORAGE_CONTAINER", None)
|
||||
allowed_extensions = app.config.get("STORAGE_ALLOWED_EXTENSIONS", None)
|
||||
serve_files = app.config.get("STORAGE_SERVER", False)
|
||||
serve_files_url = app.config.get("STORAGE_SERVER_URL", "files")
|
||||
|
||||
self.config["serve_files"] = serve_files
|
||||
self.config["serve_files_url"] = serve_files_url
|
||||
|
||||
if provider and provider.upper() == "LOCAL":
|
||||
if not local_path:
|
||||
raise LocalPathUndefinedError("For 'LOCAL' provider, Storage requires CLOUDSTORAGE_LOCAL_PATH")
|
||||
else:
|
||||
key = local_path
|
||||
secret = None
|
||||
if not provider:
|
||||
raise ValueError("'STORAGE_PROVIDER' is missing")
|
||||
|
||||
if provider.upper() == "LOCAL":
|
||||
if not os.path.isdir(container):
|
||||
raise IOError("Local Container (directory) '%s' is not a "
|
||||
"directory or doesn't exist for LOCAL provider" % container)
|
||||
|
||||
self.__init__(provider=provider,
|
||||
key=key,
|
||||
secret=secret,
|
||||
container=container,
|
||||
local_path=local_path,
|
||||
allowed_extensions=allowed_extensions,
|
||||
secure_url=secure_url)
|
||||
allowed_extensions=allowed_extensions)
|
||||
|
||||
self._register_file_server(app)
|
||||
|
||||
def get(self, object_name, secure_url=None):
|
||||
def get(self, object_name):
|
||||
"""
|
||||
Return an object or None if it doesn't exist
|
||||
|
||||
:param object_name:
|
||||
:param secure_url: To secure url, when get_url
|
||||
:return: Object
|
||||
"""
|
||||
if object_name in self:
|
||||
return Object(obj=self.container.get_object(object_name),
|
||||
secure_url=secure_url or self.secure_url,
|
||||
local_path=self.local_path)
|
||||
return Object(obj=self.container.get_object(object_name))
|
||||
return None
|
||||
|
||||
def create(self, object_name, secure_url=None, size=0, hash=None, extra=None, meta_data=None):
|
||||
def create(self, object_name, size=0, hash=None, extra=None, meta_data=None):
|
||||
"""
|
||||
create a new object
|
||||
|
||||
:param object_name:
|
||||
:param secure_url: To secure url, when get_url
|
||||
:param size:
|
||||
:param hash:
|
||||
:param extra:
|
||||
@@ -281,9 +246,7 @@ class Storage(object):
|
||||
hash=hash,
|
||||
extra=extra,
|
||||
meta_data=meta_data)
|
||||
return Object(obj=obj,
|
||||
secure_url=secure_url or self.secure_url,
|
||||
local_path=self.local_path)
|
||||
return Object(obj=obj)
|
||||
|
||||
def upload(self,
|
||||
file,
|
||||
@@ -291,6 +254,7 @@ class Storage(object):
|
||||
prefix=None,
|
||||
allowed_extensions=None,
|
||||
overwrite=False,
|
||||
public=False,
|
||||
**kwargs):
|
||||
"""
|
||||
To upload file
|
||||
@@ -299,9 +263,12 @@ class Storage(object):
|
||||
:param prefix: A prefix for the object. Can be in the form of directory tree
|
||||
:param allowed_extensions: list of extensions to allow
|
||||
:param overwrite: bool - To overwrite if file exists
|
||||
:param public: bool - To set acl to private or public-read. Having acl in kwargs will override it
|
||||
:param kwargs: extra params: ie: acl, meta_data etc.
|
||||
:return: Object
|
||||
"""
|
||||
if "acl" not in kwargs:
|
||||
kwargs["acl"] = "public-read" if public else "private"
|
||||
extra = kwargs
|
||||
|
||||
# coming from an upload object
|
||||
@@ -336,15 +303,13 @@ class Storage(object):
|
||||
|
||||
if isinstance(file, FileStorage):
|
||||
obj = self.container.upload_object_via_stream(iterator=file,
|
||||
object_name=name,
|
||||
extra=extra)
|
||||
object_name=name,
|
||||
extra=extra)
|
||||
else:
|
||||
obj = self.container.upload_object(file_path=file,
|
||||
object_name=name,
|
||||
extra=extra)
|
||||
return Object(obj=obj,
|
||||
secure_url=self.secure_url,
|
||||
local_path=self.local_path)
|
||||
return Object(obj=obj)
|
||||
|
||||
def _safe_object_name(self, object_name):
|
||||
""" Add a UUID if to a object name if it exists. To prevent overwrites
|
||||
@@ -373,16 +338,21 @@ class Storage(object):
|
||||
if server_url:
|
||||
url = "/%s/<path:object_name>" % server_url
|
||||
|
||||
@app.route(url, endpoint=FILE_SERVER_ENDPOINT)
|
||||
@app.route(url, endpoint=SERVER_ENDPOINT)
|
||||
def files_server(object_name):
|
||||
obj = self.get(object_name)
|
||||
if obj:
|
||||
dl = request.args.get("dl")
|
||||
name = request.args.get("name")
|
||||
_url = obj.get_cdn_url()
|
||||
return send_file(_url, conditional=True)
|
||||
return send_file(_url,
|
||||
as_attachment=True if dl else False,
|
||||
attachment_filename=name or False,
|
||||
conditional=True)
|
||||
else:
|
||||
abort(404)
|
||||
else:
|
||||
warnings.warn("Flask-CloudStorage can't serve files. 'CLOUDSTORAGE_SERVER_FILES_URL' is not set")
|
||||
warnings.warn("Flask-Cloudy can't serve files. 'STORAGE_SERVER_FILES_URL' is not set")
|
||||
|
||||
class Object(object):
|
||||
"""
|
||||
@@ -402,6 +372,9 @@ class Object(object):
|
||||
download()
|
||||
delete()
|
||||
"""
|
||||
|
||||
_obj = None
|
||||
|
||||
def __init__(self, obj, **kwargs):
|
||||
self._obj = obj
|
||||
self._kwargs = kwargs
|
||||
@@ -412,23 +385,22 @@ class Object(object):
|
||||
def __len__(self):
|
||||
return self.size
|
||||
|
||||
def get_url(self, secure=None, short=True):
|
||||
def get_url(self, secure=False, longurl=False):
|
||||
"""
|
||||
Return the url
|
||||
:param secure: bool - To use https
|
||||
:param short: bool - On local, reference the local path without the domain
|
||||
ie: http://site.com/files/object.png -> /files/object.png
|
||||
:param longurl: bool - On local, reference the local path with the domain
|
||||
ie: http://site.com/files/object.png otherwise /files/object.png
|
||||
:return: str
|
||||
"""
|
||||
secure = secure or self._kwargs.get("secure_url", False)
|
||||
driver_name = self.driver.name.lower()
|
||||
try:
|
||||
# Currently only Cloudfiles and Local supports it
|
||||
url = self._obj.get_cdn_url()
|
||||
if "local" in driver_name:
|
||||
url = url_for(FILE_SERVER_ENDPOINT,
|
||||
url = url_for(SERVER_ENDPOINT,
|
||||
object_name=self.name,
|
||||
_external=False if short else True)
|
||||
_external=longurl)
|
||||
except NotImplementedError as e:
|
||||
object_path = '%s/%s' % (self.container.name, self.name)
|
||||
if 's3' in driver_name:
|
||||
@@ -462,6 +434,32 @@ class Object(object):
|
||||
url = url.replace('http://', 'https://')
|
||||
return url
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
"""
|
||||
Returns the url of the object.
|
||||
For local it will return it with the domain name
|
||||
:return:
|
||||
"""
|
||||
return self.get_url(longurl=True)
|
||||
|
||||
@property
|
||||
def short_url(self):
|
||||
"""
|
||||
Returns the url of the object
|
||||
For local it will return it WITHOUT the domain name
|
||||
:return:
|
||||
"""
|
||||
return self.get_url()
|
||||
|
||||
@property
|
||||
def secure_url(self):
|
||||
"""
|
||||
Return a url with https
|
||||
:return:
|
||||
"""
|
||||
return self.get_url(secure=True, longurl=True)
|
||||
|
||||
@property
|
||||
def extension(self):
|
||||
"""
|
||||
@@ -487,29 +485,45 @@ class Object(object):
|
||||
return get_provider_name(self.driver)
|
||||
|
||||
@property
|
||||
def container_name(self):
|
||||
"""
|
||||
Return the container name
|
||||
:return: str
|
||||
"""
|
||||
return self.container.name
|
||||
|
||||
@property
|
||||
def local_path(self):
|
||||
"""
|
||||
Return the local path for Local storage
|
||||
:return: str
|
||||
"""
|
||||
return self._kwargs.get("local_path", None)
|
||||
|
||||
@property
|
||||
def object_path(self):
|
||||
def path(self):
|
||||
"""
|
||||
Return the object path
|
||||
:return: str
|
||||
"""
|
||||
path = "%s/%s" % (self.container.name, self.name)
|
||||
if "local" in self.driver.name.lower():
|
||||
path = "%s/%s" % (self.local_path, path)
|
||||
return path
|
||||
return "%s/%s" % (self.container.name, self.name)
|
||||
|
||||
def download(self, name=None, signed=True):
|
||||
"""
|
||||
Trigger a browse download
|
||||
:return:
|
||||
"""
|
||||
if "local" in self.driver.name.lower():
|
||||
return url_for(SERVER_ENDPOINT,
|
||||
object_name=self.path,
|
||||
dl=1,
|
||||
name=name)
|
||||
else:
|
||||
if signed:
|
||||
pass
|
||||
pass
|
||||
|
||||
def save_to(self, destination, name=None, overwrite=False, delete_on_failure=True):
|
||||
"""
|
||||
To save the object in a local path
|
||||
:param destination: str - The directory to save the object to
|
||||
:param name: str - To rename the file name. Do not add extesion
|
||||
:param overwrite:
|
||||
:param delete_on_failure:
|
||||
:return: The new location of the file or None
|
||||
"""
|
||||
if not os.path.isdir(destination):
|
||||
raise IOError("'%s' is not a valid directory")
|
||||
|
||||
obj_path = "%s/%s" % (destination, self._obj.name)
|
||||
if name:
|
||||
obj_path = "%s/%s.%s" % (destination, name, self.extension)
|
||||
|
||||
file = self._obj.download(obj_path,
|
||||
overwrite_existing=overwrite,
|
||||
delete_on_failure=delete_on_failure)
|
||||
return obj_path if file else None
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Flask-CloudStorage
|
||||
Flask-Cloudy
|
||||
|
||||
A wrapper around Apache-Libcloud to upload and save files on cloud storage
|
||||
providers such as: AWS S3, Google Storage, Microsoft Azure, Rackspace Cloudfiles,
|
||||
@@ -18,9 +18,8 @@ Supported storage:
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
__NAME__ = "Flask-CloudStorage"
|
||||
__version__ = "0.6.0"
|
||||
__NAME__ = "Flask-Cloudy"
|
||||
__version__ = "0.7.0"
|
||||
__author__ = "Mardix"
|
||||
__license__ = "MIT"
|
||||
__copyright__ = "2015"
|
||||
@@ -31,11 +30,11 @@ setup(
|
||||
license=__license__,
|
||||
author=__author__,
|
||||
author_email='mardix@github.com',
|
||||
description="Flask-CloudStorage is a simple flask extension and standalone library to upload and save files on the cloud",
|
||||
description="Flask-Cloudy is a simple flask extension and standalone library to upload and save files on S3, Google storage or other Cloud Storages",
|
||||
long_description=__doc__,
|
||||
url='https://github.com/mardix/flask-cloudstorage/',
|
||||
download_url='http://github.com/mardix/flask-cloudstorage/tarball/master',
|
||||
py_modules=['flask_cloudstorage'],
|
||||
url='https://github.com/mardix/flask-cloudy/',
|
||||
download_url='http://github.com/mardix/flask-cloudy/tarball/master',
|
||||
py_modules=['flask_cloudy'],
|
||||
include_package_data=True,
|
||||
packages=find_packages(),
|
||||
install_requires=[
|
||||
@@ -47,7 +46,7 @@ setup(
|
||||
'python-slugify==0.1.0'
|
||||
],
|
||||
|
||||
keywords=["flask", "s3", "aws", "cloudfiles", "storage", "azure", "google"],
|
||||
keywords=["flask", "s3", "aws", "cloudfiles", "storage", "azure", "google", "cloudy"],
|
||||
platforms='any',
|
||||
classifiers=[
|
||||
'Environment :: Web Environment',
|
||||
|
||||
+2
-6
@@ -1,14 +1,10 @@
|
||||
|
||||
# ON S3
|
||||
|
||||
PROVIDER = "S3"
|
||||
KEY = ""
|
||||
SECRET = ""
|
||||
CONTAINER = ""
|
||||
CONTAINER_2 = ""
|
||||
|
||||
CONTAINER = "yoredis.com"
|
||||
|
||||
# FOR LOCAL
|
||||
PROVIDER = "LOCAL"
|
||||
CONTAINER = "container_1"
|
||||
CONTAINER_2 = "container_2"
|
||||
LOCAL_PATH = ""
|
||||
@@ -1,10 +1,7 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from libcloud.storage.base import (StorageDriver,
|
||||
Container)
|
||||
from tests import config
|
||||
from flask_cloudstorage import (get_file_extension,
|
||||
from libcloud.storage.base import (StorageDriver, Container)
|
||||
from flask_cloudy import (get_file_extension,
|
||||
get_file_extension_type,
|
||||
get_file_name,
|
||||
get_driver_class,
|
||||
@@ -12,18 +9,20 @@ from flask_cloudstorage import (get_file_extension,
|
||||
Storage,
|
||||
Object,
|
||||
InvalidExtensionError)
|
||||
from tests import config
|
||||
|
||||
CWD = os.path.dirname(__file__)
|
||||
|
||||
# Manipulate
|
||||
CONTAINER = "%s/%s" % (CWD, config.CONTAINER) if config.PROVIDER == "LOCAL" else config.CONTAINER
|
||||
|
||||
class App(object):
|
||||
config = config=dict(
|
||||
CLOUDSTORAGE_PROVIDER=config.PROVIDER,
|
||||
CLOUDSTORAGE_KEY=config.KEY,
|
||||
CLOUDSTORAGE_SECRET=config.SECRET,
|
||||
CLOUDSTORAGE_CONTAINER=config.CONTAINER,
|
||||
CLOUDSTORAGE_LOCAL_PATH=CWD,
|
||||
CLOUDSTORAGE_ALLOWED_EXTENSIONS=[])
|
||||
config = dict(
|
||||
STORAGE_PROVIDER=config.PROVIDER,
|
||||
STORAGE_KEY=config.KEY,
|
||||
STORAGE_SECRET=config.SECRET,
|
||||
STORAGE_CONTAINER=CONTAINER,
|
||||
STORAGE_ALLOWED_EXTENSIONS=[])
|
||||
|
||||
|
||||
def _setup_function():
|
||||
pass
|
||||
@@ -49,13 +48,12 @@ def test_get_provider_name():
|
||||
driver = GoogleStorageDriver()
|
||||
assert get_provider_name(driver) == "google_storage"
|
||||
|
||||
|
||||
#---
|
||||
|
||||
app = App()
|
||||
|
||||
def app_storage():
|
||||
return Storage(app=app)
|
||||
return Storage(app=App())
|
||||
|
||||
def test_get_driver_class():
|
||||
driver = get_driver_class("S3")
|
||||
@@ -69,11 +67,6 @@ def test_container():
|
||||
storage = app_storage()
|
||||
assert isinstance(storage.container, Container)
|
||||
|
||||
def test_set_container():
|
||||
storage = app_storage()
|
||||
storage.container = config.CONTAINER_2
|
||||
assert storage.container.name == config.CONTAINER_2
|
||||
|
||||
def test_flask_app():
|
||||
storage = app_storage()
|
||||
assert isinstance(storage.driver, StorageDriver)
|
||||
@@ -107,25 +100,12 @@ def test_object_provider_name():
|
||||
o = storage.create(object_name)
|
||||
assert o.provider_name == config.PROVIDER.lower()
|
||||
|
||||
def test_object_container_name():
|
||||
object_name = "hello.jpg"
|
||||
storage = app_storage()
|
||||
o = storage.create(object_name)
|
||||
assert o.container_name == config.CONTAINER
|
||||
|
||||
def test_object_object_path():
|
||||
object_name = "hello.jpg"
|
||||
storage = app_storage()
|
||||
o = storage.create(object_name)
|
||||
p = "%s/%s" % (o.container.name, o.name)
|
||||
assert o.object_path.endswith(p)
|
||||
|
||||
def test_object_local_path():
|
||||
object_name = "hello.jpg"
|
||||
storage = app_storage()
|
||||
o = storage.create(object_name)
|
||||
if "local" in o.container.name.lower():
|
||||
assert o.local_path == CWD
|
||||
assert o.path.endswith(p)
|
||||
|
||||
def test_storage_upload_invalid():
|
||||
storage = app_storage()
|
||||
@@ -180,3 +160,13 @@ def test_storage_upload_with_prefix():
|
||||
o = storage.upload(CWD + "/data/hello.txt", name=object_name, prefix=prefix, overwrite=True)
|
||||
assert full_name in storage
|
||||
assert o.name == full_name
|
||||
|
||||
|
||||
def test_save_to():
|
||||
storage = app_storage()
|
||||
object_name = "my-txt-hello-to-save.txt"
|
||||
o = storage.upload(CWD + "/data/hello.txt", name=object_name)
|
||||
file = o.save_to("./data", overwrite=True)
|
||||
file2 = o.save_to(CWD + "/data", name="my_new_file", overwrite=True)
|
||||
assert os.path.isfile(file)
|
||||
assert file2 == CWD + "/data/my_new_file.txt"
|
||||
Reference in New Issue
Block a user