From be645e012b2f0835c9676ae89d9f3cf0eafc077b Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 9 Nov 2011 17:41:15 -0600 Subject: [PATCH] First attempt at multithreaded python 2.7 version of downloader. --- app.yaml | 22 +++++++++++++++------ main.py | 51 ++++++++++++++++++++++-------------------------- utils/remover.py | 25 +++++++++++------------- utils/tally.py | 21 ++++++++------------ 4 files changed, 58 insertions(+), 61 deletions(-) diff --git a/app.yaml b/app.yaml index f6428d3..7c2010a 100644 --- a/app.yaml +++ b/app.yaml @@ -1,20 +1,24 @@ # fanfictionloader ffd-retief application: fanfictionloader -version: 4-0-7 -runtime: python +version: 4-1-0 +runtime: python27 api_version: 1 +threadsafe: true handlers: +- url: /hello + script: utils.helloworld.app + - url: /r3m0v3r.* - script: utils/remover.py + script: utils.remover.app login: admin - url: /tally.* - script: utils/tally.py + script: utils.tally.app login: admin - url: /fdownloadtask - script: main.py + script: main.app login: admin - url: /css @@ -31,7 +35,13 @@ handlers: upload: static/favicon\.ico - url: /.* - script: main.py + script: main.app builtins: - datastore_admin: on + +libraries: +- name: django + version: "1.2" +- name: PIL + version: "1.1.7" diff --git a/main.py b/main.py index 4240b4a..50da665 100644 --- a/main.py +++ b/main.py @@ -41,23 +41,24 @@ import ConfigParser ## Console page first, you will get a django version mismatch error when you ## to go hit one of the application pages. Just change a file again, and ## make sure to hit an app page before the SDK page to clear it. -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' -from google.appengine.dist import use_library -use_library('django', '1.2') +#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' +#from google.appengine.dist import use_library +#use_library('django', '1.2') from google.appengine.ext import db from google.appengine.api import taskqueue from google.appengine.api import users -from google.appengine.ext import webapp +#from google.appengine.ext import webapp +import webapp2 from google.appengine.ext.webapp import template -from google.appengine.ext.webapp import util +#from google.appengine.ext.webapp2 import util from google.appengine.runtime import DeadlineExceededError from ffstorage import * from fanficdownloader import adapters, writers, exceptions -class UserConfigServer(webapp.RequestHandler): +class UserConfigServer(webapp2.RequestHandler): def getUserConfig(self,user): config = ConfigParser.SafeConfigParser() @@ -73,7 +74,7 @@ class UserConfigServer(webapp.RequestHandler): return config -class MainHandler(webapp.RequestHandler): +class MainHandler(webapp2.RequestHandler): def get(self): user = users.get_current_user() if user: @@ -160,7 +161,7 @@ class EditConfigServer(UserConfigServer): self.response.out.write(template.render(path, template_values)) -class FileServer(webapp.RequestHandler): +class FileServer(webapp2.RequestHandler): def get(self): fileId = self.request.get('id') @@ -221,7 +222,7 @@ class FileServer(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'status.html') self.response.out.write(template.render(path, template_values)) -class FileStatusServer(webapp.RequestHandler): +class FileStatusServer(webapp2.RequestHandler): def get(self): user = users.get_current_user() if not user: @@ -257,7 +258,7 @@ class FileStatusServer(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'status.html') self.response.out.write(template.render(path, template_values)) -class ClearRecentServer(webapp.RequestHandler): +class ClearRecentServer(webapp2.RequestHandler): def get(self): user = users.get_current_user() if not user: @@ -282,7 +283,7 @@ class ClearRecentServer(webapp.RequestHandler): logging.info('Deleted %d instances download.' % num) self.redirect("/?error=recentcleared") -class RecentFilesServer(webapp.RequestHandler): +class RecentFilesServer(webapp2.RequestHandler): def get(self): user = users.get_current_user() if not user: @@ -556,20 +557,14 @@ def urlEscape(data): p = re.compile(r'([^\w])') return p.sub(toPercentDecimal, data.encode("utf-8")) -def main(): - application = webapp.WSGIApplication([('/', MainHandler), - ('/fdowntask', FanfictionDownloaderTask), - ('/fdown', FanfictionDownloader), - (r'/file.*', FileServer), - ('/status', FileStatusServer), - ('/recent', RecentFilesServer), - ('/editconfig', EditConfigServer), - ('/clearrecent', ClearRecentServer), - ], - debug=False) - util.run_wsgi_app(application) - - -if __name__ == '__main__': - logging.getLogger().setLevel(logging.DEBUG) - main() +logging.getLogger().setLevel(logging.DEBUG) +app = webapp2.WSGIApplication([('/', MainHandler), + ('/fdowntask', FanfictionDownloaderTask), + ('/fdown', FanfictionDownloader), + (r'/file.*', FileServer), + ('/status', FileStatusServer), + ('/recent', RecentFilesServer), + ('/editconfig', EditConfigServer), + ('/clearrecent', ClearRecentServer), + ], + debug=False) diff --git a/utils/remover.py b/utils/remover.py index 23e4b9b..50101d3 100644 --- a/utils/remover.py +++ b/utils/remover.py @@ -25,15 +25,16 @@ Copyright 2011 Fanficdownloader team import datetime import logging -from google.appengine.ext.webapp import util -from google.appengine.ext import webapp +#from google.appengine.ext.webapp import util +import webapp2 +#from google.appengine.ext import webapp from google.appengine.api import users from google.appengine.api import taskqueue from google.appengine.api import memcache from ffstorage import * -class Remover(webapp.RequestHandler): +class Remover(webapp2.RequestHandler): def get(self): logging.debug("Starting r3m0v3r") user = users.get_current_user() @@ -56,9 +57,10 @@ class Remover(webapp.RequestHandler): logging.debug('Delete '+d.url) logging.info('Deleted instances: %d' % num) + self.response.headers['Content-Type'] = 'text/html' self.response.out.write('Deleted instances: %d
' % num) -class RemoveOrphanDataChunks(webapp.RequestHandler): +class RemoveOrphanDataChunks(webapp2.RequestHandler): def get(self): logging.debug("Starting RemoveOrphanDataChunks") @@ -98,15 +100,10 @@ class RemoveOrphanDataChunks(webapp.RequestHandler): memcache.set('orphan_search_cursor',chunks.cursor()) logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num)) + self.response.headers['Content-Type'] = 'text/html' self.response.out.write('Deleted %d orphan chunks from %d total.' % (deleted,num)) -def main(): - application = webapp.WSGIApplication([('/r3m0v3r', Remover), - ('/r3m0v3rOrphans', RemoveOrphanDataChunks)], - debug=False) - util.run_wsgi_app(application) - - -if __name__ == '__main__': - logging.getLogger().setLevel(logging.DEBUG) - main() +logging.getLogger().setLevel(logging.DEBUG) +app = webapp2.WSGIApplication([('/r3m0v3r', Remover), + ('/r3m0v3rOrphans', RemoveOrphanDataChunks)], + debug=False) diff --git a/utils/tally.py b/utils/tally.py index 7c1b6f0..d3305f5 100644 --- a/utils/tally.py +++ b/utils/tally.py @@ -18,15 +18,16 @@ import datetime import logging -from google.appengine.ext.webapp import util -from google.appengine.ext import webapp +#from google.appengine.ext.webapp import util +import webapp2 +#from google.appengine.ext import webapp from google.appengine.api import users from google.appengine.api import taskqueue from google.appengine.api import memcache from ffstorage import * -class Tally(webapp.RequestHandler): +class Tally(webapp2.RequestHandler): def get(self): logging.debug("Starting Tally") user = users.get_current_user() @@ -57,13 +58,7 @@ class Tally(webapp.RequestHandler): logging.info('Tallied %d fics.' % num) self.response.out.write('
Tallied %d fics.
' % num) -def main(): - application = webapp.WSGIApplication([('/tally', Tally), - ], - debug=False) - util.run_wsgi_app(application) - - -if __name__ == '__main__': - logging.getLogger().setLevel(logging.DEBUG) - main() +logging.getLogger().setLevel(logging.DEBUG) +app = webapp2.WSGIApplication([('/tally', Tally), + ], + debug=False)