From 22d0989d0ab4acdf32664a21e205aa06b02513c8 Mon Sep 17 00:00:00 2001 From: wsuetholz Date: Thu, 18 Nov 2010 16:07:58 -0600 Subject: [PATCH] Raise exceptions instead of an exit when errors are detected. Added static functions to the writer classes that contain the type name and type extension to use when writing out the destination file. Added a function to the zipdir that will check the date the fic was last updated in the current archive of a fic against the current last updated date and return whether the fic is newer then the archive file. This is not useful at this time for when this is running under appEngine. Added a flag to indicate if the program is running from appEngine or not. --- adapter.py | 20 +++++++-- downloader.py | 86 ++++++++++++++++++++++++++++++------- ffnet.py | 40 ++++++++++++----- fictionalley.py | 45 ++++++++++++++----- ficwad.py | 48 +++++++++++++++++---- fpcom.py | 39 +++++++++++++---- hpfiction.py | 39 ++++++++++++++--- mediaminer.py | 62 +++++++++++++++------------ output.py | 102 ++++++++++++++++++++++++++++---------------- potionsNsnitches.py | 53 ++++++++++++++++++----- twilighted.py | 53 ++++++++++++++++++----- zipdir.py | 68 +++++++++++++++++++++++++++++ 12 files changed, 506 insertions(+), 149 deletions(-) diff --git a/adapter.py b/adapter.py index 1360824..f4dd116 100644 --- a/adapter.py +++ b/adapter.py @@ -2,6 +2,7 @@ import logging import datetime +from output import makeAcceptableFilename try: from google.appengine.api.urlfetch import fetch as googlefetch @@ -15,6 +16,18 @@ class LoginRequiredException(Exception): def __str__(self): return repr(self.url + ' requires user to be logged in') + +class StoryArchivedAlready(Exception): + pass + +class StoryDoesNotExist(Exception): + pass + +class FailedToDownload(Exception): + pass + +class InvalidStoryURL(Exception): + pass class FanfictionSiteAdapter: appEngine = appEngineGlob @@ -93,12 +106,13 @@ class FanfictionSiteAdapter: return self.uuid def getOutputName(self): - self.outputName = self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId + self.outputName = makeAcceptableFilename(self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId) logging.debug('self.outputName=%s' % self.outputName) return self.outputName - def getOutputFileName(self, booksDirectory, format): - self.outputFileName = booksDirectory + "/" + self.getOutputName() + "." + format + def getOutputFileName(self, booksDirectory, bookExt): + self.getOutputName() # make sure self.outputName is populated + self.outputFileName = booksDirectory + "/" + self.outputName + bookExt logging.debug('self.outputFileName=%s' % self.outputFileName) return self.outputFileName diff --git a/downloader.py b/downloader.py index ee0120f..3dfd7fc 100644 --- a/downloader.py +++ b/downloader.py @@ -14,12 +14,18 @@ import urlparse as up import BeautifulSoup as bs import htmlentitydefs as hdefs +import zipdir +import output +import adapter +from adapter import StoryArchivedAlready +from adapter import StoryDoesNotExist +from adapter import FailedToDownload +from adapter import InvalidStoryURL +from adapter import LoginRequiredException import ffnet import fpcom import ficwad -import output -import adapter import fictionalley import hpfiction import twilighted @@ -31,15 +37,30 @@ import time class FanficLoader: '''A controller class which handles the interaction between various specific downloaders and writers''' booksDirectory = "books" + standAlone = False - def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True): + def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True, overwrite=False): self.adapter = adapter self.writerClass = writerClass self.quiet = quiet self.inmemory = inmemory self.compress = compress self.badLogin = False - self.overWrite = True + self.overWrite = overwrite + + def getBooksDirectory(self): + return self.booksDirectory + + def setBooksDirectory(self, bd): + self.booksDirectory = bd + return self.booksDirectory + + def getStandAlone(self): + return self.standAlone + + def setStandAlone(self, sa): + self.standAlone = sa + return self.standAlone def getAdapter(): return self.adapter @@ -55,13 +76,16 @@ class FanficLoader: urls = self.adapter.extractIndividualUrls() - if (self.adapter.hasAppEngine): - self.overWrite = True + logging.debug("self.writerClass=%s" % self.writerClass) + if self.standAlone and not self.inmemory: + s = self.adapter.getOutputFileName(self.booksDirectory, self.writerClass.getFormatExt()) + logging.debug("Always overwrite? %s" % self.overWrite) + if not self.overWrite: + logging.debug("Checking if current archive of the story exists. Filename=%s" % s) + if not zipdir.checkNewer ( s, self.adapter.getStoryUpdated() ): + raise StoryArchivedAlready("A Current archive file \"" + s + "\" already exists! Skipping!") else: - s = self.adapter.getOutputFileName(self.booksDirectory, format) - if not self.overWrite and os.path.isfile(s): - print >> sys.stderr, "File " + s + " already exists! Skipping!" - exit(10) + logging.debug("Do not check for existance of archive file.") self.writer = self.writerClass(self.booksDirectory, self.adapter, inmemory=self.inmemory, compress=self.compress) @@ -83,10 +107,17 @@ class FanficLoader: if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - (url, format) = sys.argv[1:] - # (url) = sys.argv[1] - # format = 'epub' + argvlen = len(sys.argv) + url = None + if argvlen > 1: + url = sys.argv[1] + if argvlen > 2: + bookFormat = sys.argv[2] + if url is None: + print >> sys.stderr, "Usage: downloader.py URL Type" + sys.exit(-1) + if type(url) is unicode: print('URL is unicode') url = url.encode('latin1') @@ -117,9 +148,9 @@ if __name__ == '__main__': print >> sys.stderr, "Oi! I can haz not appropriate adapter for URL %s!" % url sys.exit(1) - if format == 'epub': + if bookFormat == 'epub': writerClass = output.EPubFanficWriter - elif format == 'html': + elif bookFormat == 'html': writerClass = output.HTMLWriter if adapter.requiresLogin(url): @@ -134,5 +165,28 @@ if __name__ == '__main__': loader = FanficLoader(adapter, writerClass) - loader.download() + loader.setStandAlone(True) + + try: + loader.download() + except FailedToDownload, ftd: + print >> sys.stderr, str(ftd) + sys.exit(2) # Error Downloading + except InvalidStoryURL, isu: + print >> sys.stderr, str(isu) + sys.exit(3) # Unknown Error + except StoryArchivedAlready, se: + print >> sys.stderr, str(se) + sys.exit(10) # Skipped + except StoryDoesNotExist, sdne: + print >> sys.stderr, str(sdne) + sys.exit(20) # Missing + except LoginRequiredException, lre: + print >> sys.stderr, str(lre) + sys.exit(30) # Missing + except Exception, e: + print >> sys.stderr, str(e) + sys.exit(99) # Unknown Error + + sys.exit(0) diff --git a/ffnet.py b/ffnet.py index 7a201e2..a6fee2d 100644 --- a/ffnet.py +++ b/ffnet.py @@ -70,8 +70,7 @@ class FFNet(FanfictionSiteAdapter): logging.debug('spl=%s' % spl) if spl is not None: if len(spl) > 0 and spl[0] != 's': - logging.error("Error URL \"%s\" is not a story." % self.url) - exit (20) + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) if len(spl) > 1: self.storyId = spl[1] if len(spl) > 2: @@ -150,9 +149,23 @@ class FFNet(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = '' + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + d2 = re.sub('&\#[0-9]+;', ' ', data) - soup = bs.BeautifulStoneSoup(d2) + soup = None + try: + soup = bs.BeautifulStoneSoup(d2) + except: + logging.error("Failed to decode: <%s>" % d2) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + allA = soup.findAll('a') for a in allA: if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1: @@ -264,7 +277,15 @@ class FFNet(FanfictionSiteAdapter): def getText(self, url): time.sleep( 2.0 ) - data = self.fetchUrl(url) + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + lines = data.split('\n') textbuf = '' @@ -276,16 +297,15 @@ class FFNet(FanfictionSiteAdapter): except: data = olddata + soup = None try: soup = bs.BeautifulStoneSoup(data) except: - logging.info("Failed to decode: <%s>" % data) - soup = None + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit (20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/fictionalley.py b/fictionalley.py index fbbe95e..31f43b5 100644 --- a/fictionalley.py +++ b/fictionalley.py @@ -124,25 +124,36 @@ class FictionAlley(FanfictionSiteAdapter): def extractIndividualUrls(self): - data = self.opener.open(self.url).read() + data = '' + try: + data = self.opener.open(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") # There is some usefull information in the headers of the first chapter page.. data = data.replace('','').replace('','') - soup = bs.BeautifulStoneSoup(data) + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) breadcrumbs = soup.find('div', {'class': 'breadcrumbs'}) if breadcrumbs is not None: # Be aware that this means that the user has entered the {STORY}01.html # We will not have valid Publised and Updated dates. User should enter # the {STORY}.html instead. We should force that instead of this. - logging.debug('breadcrumbs=%s' % breadcrumbs ) + #logging.debug('breadcrumbs=%s' % breadcrumbs ) bcas = breadcrumbs.findAll('a') - logging.debug('bcas=%s' % bcas ) + #logging.debug('bcas=%s' % bcas ) if bcas is not None and len(bcas) > 1: bca = bcas[1] - logging.debug('bca=%s' % bca ) + #logging.debug('bca=%s' % bca ) if 'href' in bca._getAttrMap(): - logging.debug('bca.href=%s' % bca['href'] ) + #logging.debug('bca.href=%s' % bca['href'] ) url = str(bca['href']) if url is not None and len(url) > 0: self.url = url @@ -244,7 +255,15 @@ class FictionAlley(FanfictionSiteAdapter): def getText(self, url): # fictionalley uses full URLs in chapter list. - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + # find & and # replaced with matching div pair for easier parsing. @@ -252,13 +271,17 @@ class FictionAlley(FanfictionSiteAdapter): # something other than div prevents soup from pairing # our div with poor html inside the story text. data = data.replace('','').replace('','') - soup = bs.BeautifulStoneSoup(data) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) div = soup.find('crazytagstringnobodywouldstumbleonaccidently', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) html = soup.findAll('html') if len(html) > 1: diff --git a/ficwad.py b/ficwad.py index 7e9f3ae..a2aab78 100644 --- a/ficwad.py +++ b/ficwad.py @@ -57,9 +57,21 @@ class FicWad(FanfictionSiteAdapter): def extractIndividualUrls(self): oldurl = '' - - data = u2.urlopen(self.url).read() - soup = bs.BeautifulStoneSoup(data) + cururl = self.url + data = '' + try: + data = u2.urlopen(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) story = soup.find('div', {'id' : 'story'}) crumbtrail = story.find('h3') # the only h3 ficwad uses. @@ -100,7 +112,7 @@ class FicWad(FanfictionSiteAdapter): meta = soup.find('p', {'class' : 'meta'}) if meta is not None: s = str(meta).replace('\n',' ').replace('\t','').split(' - ') - logging.debug('meta.s=%s' % s) + #logging.debug('meta.s=%s' % s) for ss in s: s1 = ss.replace(' ','').split(':') #logging.debug('meta.s.s1=%s' % s1) @@ -164,11 +176,18 @@ class FicWad(FanfictionSiteAdapter): ii = 1 if oldurl is not None and len(oldurl) > 0: + logging.debug('Switching back to %s' % oldurl) + cururl = oldurl data = u2.urlopen(oldurl).read() soup = bs.BeautifulStoneSoup(data) storylist = soup.find('ul', {'id' : 'storylist'}) if storylist is not None: + allBlocked = storylist.findAll('li', {'class' : 'blocked'}) + if allBlocked is not None: + #logging.debug('allBlocked=%s' % allBlocked) + raise LoginRequiredException(cururl) + allH4s = storylist.findAll('h4') #logging.debug('allH4s=%s' % allH4s) @@ -216,14 +235,25 @@ class FicWad(FanfictionSiteAdapter): if url.find('http://') == -1: url = 'http://' + self.host + '/' + url - data = u2.urlopen(url).read() + data = '' + try: + data = u2.urlopen(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + try: + soup = bs.BeautifulStoneSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - soup = bs.BeautifulStoneSoup(data) div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + return div.__str__('utf8') diff --git a/fpcom.py b/fpcom.py index b943132..3598c2e 100644 --- a/fpcom.py +++ b/fpcom.py @@ -65,8 +65,7 @@ class FPCom(FanfictionSiteAdapter): spl = self.path.split('/') if spl is not None: if len(spl) > 0 and spl[0] != 's': - logging.error("Error URL \"%s\" is not a story." % self.url) - exit (20) + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) if len(spl) > 1: self.storyId = spl[1] if len(spl) > 2: @@ -138,9 +137,23 @@ class FPCom(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = '' + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + d2 = re.sub('&\#[0-9]+;', ' ', data) - soup = bs.BeautifulStoneSoup(d2) + soup = None + try: + soup = bs.BeautifulStoneSoup(d2) + except: + logging.error("Failed to decode: <%s>" % d2) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + allA = soup.findAll('a') for a in allA: if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1: @@ -277,7 +290,15 @@ class FPCom(FanfictionSiteAdapter): def getText(self, url): time.sleep( 2.0 ) - data = self.fetchUrl(url) + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + lines = data.split('\n') textbuf = '' @@ -289,16 +310,16 @@ class FPCom(FanfictionSiteAdapter): except: data = olddata + soup = None try: soup = bs.BeautifulStoneSoup(data) except: logging.info("Failed to decode: <%s>" % data) - soup = None + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit (20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/hpfiction.py b/hpfiction.py index 3850939..b2116ef 100644 --- a/hpfiction.py +++ b/hpfiction.py @@ -84,10 +84,21 @@ class HPFiction(FanfictionSiteAdapter): return self.path def extractIndividualUrls(self): + data = '' + try: + data = self.opener.open(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") - data = self.opener.open(self.url).read() - soup = bs.BeautifulSoup(data) - + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + links = soup.findAll('a') def_chapurl = '' def_chaptitle = '' @@ -220,12 +231,26 @@ class HPFiction(FanfictionSiteAdapter): def getText(self, url): logging.debug('Downloading from URL: %s' % url) - data = self.opener.open(url).read() - soup = bs.BeautifulSoup(data) + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + divtext = soup.find('div', {'id' : 'fluidtext'}) if None == divtext: - logging.error("Error downloading Chapter: %s" % url) - exit(20) + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + return divtext.__str__('utf8') diff --git a/mediaminer.py b/mediaminer.py index e8389ab..2b790f4 100644 --- a/mediaminer.py +++ b/mediaminer.py @@ -81,7 +81,9 @@ class MediaMiner(FanfictionSiteAdapter): self.storyId = ss[-2].strip() self.path = '/fanfic/view_st.php/' + self.storyId self.url = 'http://' + self.host + self.path - logging.debug('self.url=%s' % self.url) + logging.debug('self.url=%s' % self.url) + else: + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) logging.debug('self.storyId=%s' % self.storyId) @@ -144,9 +146,23 @@ class MediaMiner(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = None + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = None + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + #data.replace('
',' ').replace('
',' ').replace('
',' ') - soup = bs.BeautifulSoup(data) + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + logging.error("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + #logging.debug('soap=%s' % soup) urls = [] @@ -175,7 +191,7 @@ class MediaMiner(FanfictionSiteAdapter): for ii in range(ll): td = td_smtxt[ii] if 'class' in td._getAttrMap() and td['class'] != 'smtxt': - logging.debug('td has class attribute but is not smtxt') + #logging.debug('td has class attribute but is not smtxt') continue ss = str(td).replace('\n','').replace('\r','').replace(' ', ' ') #logging.debug('ss=%s' % ss) @@ -309,31 +325,28 @@ class MediaMiner(FanfictionSiteAdapter): self.numChapters = str(numchapters) logging.debug('self.numChapters=%s' % self.numChapters) - logging.debug('urls=%s' % urls) + #logging.debug('urls=%s' % urls) return urls def getText(self, url): time.sleep( 2.0 ) logging.debug('url=%s' % url) - data = self.fetchUrl(url) - + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None try: soup = bs.BeautifulSoup(data) except: - logging.info("Failed to decode: <%s>" % data) - soup = None - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - #div = soup.find('div', {'id' : 'storytext'}) - #if div is None: - #logging.error("Error downloading Chapter: %s" % url) - #exit (20) - #return '' - - #logging.info("Soup: %s" % soup.prettify()) - nvs = bs.NavigableString('') sst='' allAs = soup.findAll ('a', { 'name' : 'fic_c' }) @@ -368,16 +381,9 @@ class MediaMiner(FanfictionSiteAdapter): sst = sst + st nxta = nxta.nextSibling - #sst = sst.replace(' ',' ').strip() - #logging.debug('sst=%s' % sst) + if sst is None: + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - #logging.debug('sst.0=%s' % sst) - #sst0 = sst.replace(u'≴', u'“').replace(u'≵','”').replace(u'≰',u'‘').replace(u'≱',u'’') - #sst0 = sst.replace(u"≵","”") - #logging.debug('sst.1=%s' % sst0) - #sst1 = sst.replace(u'≴', u'\"').replace('≵','\"').replace('≰','\'').replace('≱','\'') - #logging.debug('sst.2=%s' % sst1) - return sst class FPC_UnitTests(unittest.TestCase): diff --git a/output.py b/output.py index a320a42..64a64d0 100644 --- a/output.py +++ b/output.py @@ -40,9 +40,25 @@ class FanficWriter: def finalise(self): pass + @staticmethod + def getFormatName(): + return 'base' + + @staticmethod + def getFormatExt(): + return '.bse' + class TextWriter(FanficWriter): htmlWriter = None + @staticmethod + def getFormatName(): + return 'text' + + @staticmethod + def getFormatExt(): + return '.txt' + def __init__(self, base, adapter, inmemory=False, compress=False): self.htmlWriter = HTMLWriter(base, adapter, True, False) @@ -59,11 +75,19 @@ class TextWriter(FanficWriter): class HTMLWriter(FanficWriter): body = '' + @staticmethod + def getFormatName(): + return 'html' + + @staticmethod + def getFormatExt(): + return '.html' + def __init__(self, base, adapter, inmemory=False, compress=False): self.basePath = base self.storyTitle = removeEntities(adapter.getStoryName()) self.name = makeAcceptableFilename(adapter.getOutputName()) - self.fileName = self.basePath + '/' + self.name + '.html' + self.fileName = self.basePath + '/' + self.name + self.getFormatExt() self.authorName = removeEntities(adapter.getAuthorName()) self.adapter = adapter @@ -111,6 +135,45 @@ class EPubFanficWriter(FanficWriter): files = {} + @staticmethod + def getFormatName(): + return 'epub' + + @staticmethod + def getFormatExt(): + return '.epub' + + def __init__(self, base, adapter, inmemory=False, compress=True): + self.basePath = base + self.storyTitle = removeEntities(adapter.getStoryName()) + self.name = makeAcceptableFilename(adapter.getOutputName()) + self.directory = self.basePath + '/' + self.name + self.authorName = removeEntities(adapter.getAuthorName()) + self.inmemory = inmemory + self.adapter = adapter + + self.files = {} + self.chapters = [] + + if not self.inmemory: + self.inmemory = True + self.writeToFile = True + else: + self.writeToFile = False + + if not self.inmemory: + if os.path.exists(self.directory): + shutil.rmtree(self.directory) + + os.mkdir(self.directory) + + os.mkdir(self.directory + '/META-INF') + os.mkdir(self.directory + '/OEBPS') + + self._writeFile('mimetype', MIMETYPE) + self._writeFile('META-INF/container.xml', CONTAINER) + self._writeFile('OEBPS/stylesheet.css', CSS) + def _writeFile(self, fileName, data): #logging.debug('_writeFile(`%s`, data)' % fileName) if fileName in self.files: @@ -134,39 +197,6 @@ class EPubFanficWriter(FanficWriter): for f in self.files: self.files[f].close() - def __init__(self, base, adapter, inmemory=False, compress=True): - self.basePath = base - self.storyTitle = removeEntities(adapter.getStoryName()) - self.name = makeAcceptableFilename(adapter.getOutputName()) - self.directory = self.basePath + '/' + self.name - self.authorName = removeEntities(adapter.getAuthorName()) - self.inmemory = inmemory - self.adapter = adapter - - self.files = {} - self.chapters = [] - - if not self.inmemory: - self.inmemory = True - self.writeToFile = True - else: - self.writeToFile = False - - - - if not self.inmemory: - if os.path.exists(self.directory): - shutil.rmtree(self.directory) - - os.mkdir(self.directory) - - os.mkdir(self.directory + '/META-INF') - os.mkdir(self.directory + '/OEBPS') - - self._writeFile('mimetype', MIMETYPE) - self._writeFile('META-INF/container.xml', CONTAINER) - self._writeFile('OEBPS/stylesheet.css', CSS) - def writeChapter(self, index, title, text): title = removeEntities(title) logging.debug("Writing chapter: %s" % title) @@ -321,7 +351,7 @@ class EPubFanficWriter(FanficWriter): self._closeFiles() - filename = self.directory + '.epub' + filename = self.directory + self.getFormatExt() zipdata = zipdir.inMemoryZip(self.files) @@ -382,4 +412,4 @@ def removeEntities(text): return text def makeAcceptableFilename(text): - return re.sub('[^a-zA-Z0-9_\'-]+','',removeEntities(text).replace(" ", "_").replace(":","_")) + return re.sub('[^a-zA-Z0-9_-]+','',removeEntities(text).replace(" ", "_").replace(":","_")) diff --git a/potionsNsnitches.py b/potionsNsnitches.py index 3868e93..d0c3fa5 100644 --- a/potionsNsnitches.py +++ b/potionsNsnitches.py @@ -130,15 +130,35 @@ class PotionsNSnitches(FanfictionSiteAdapter): def extractIndividualUrls(self): url = self.url + '&chapter=1' - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") if self.reqLoginData(data): - self.performLogin() - data = self.opener.open(url).read() - if self.reqLoginData(data): - return None + self.performLogin() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") - soup = bs.BeautifulStoneSoup(data) + if self.reqLoginData(data): + raise FailedToDownload("Error downloading Story: %s! Login Failed!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % url) self.storyName = '' self.authorName = '' @@ -302,16 +322,29 @@ class PotionsNSnitches(FanfictionSiteAdapter): logging.debug('Getting data from: %s' % url) - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) # need to do this, because for some reason the
tag in the story causes problems data = data.replace('
', ' SOMETHING_BR ') - soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) - + + soup = None + try: + soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'story'}) if None == div: - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) # put the
tags back in.. text = div.__str__('utf8').replace(' SOMETHING_BR ','
') diff --git a/twilighted.py b/twilighted.py index ed29340..00f5936 100644 --- a/twilighted.py +++ b/twilighted.py @@ -110,15 +110,36 @@ class Twilighted(FanfictionSiteAdapter): def extractIndividualUrls(self): url = self.url + '&chapter=1' - data = self.opener.open(url).read() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") if self.reqLoginData(data): - self.performLogin() - data = self.opener.open(url).read() - if self.reqLoginData(data): - return None + self.performLogin() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") - soup = bs.BeautifulStoneSoup(data) + if self.reqLoginData(data): + raise FailedToDownload("Error downloading Story: %s! Login Failed!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % url) title = soup.find('title').string logging.debug('Title: %s' % title) @@ -254,14 +275,26 @@ class Twilighted(FanfictionSiteAdapter): logging.debug('Getting data from: %s' % url) - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) - div = soup.find('div', {'id' : 'story'}) if None == div: - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/zipdir.py b/zipdir.py index 597b4f8..6079e08 100644 --- a/zipdir.py +++ b/zipdir.py @@ -1,11 +1,79 @@ # -*- coding: utf-8 -*- +import sys import os +import zlib import zipfile +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED +from contextlib import closing import logging +import BeautifulSoup as bs +import htmlentitydefs as hdefs +import time +import datetime +from datetime import timedelta + import StringIO +class InvalidEPub(Exception): + pass + +def checkNewer(filename, curdte): + ret = True + + if not os.path.isfile(filename): + logging.debug('File %s does not already exist.' % filename) + return ret + + #logging.debug('filename=%s, curdte=%s' % (filename, curdte)) + lastdate = None + with closing(ZipFile(open(filename, 'rb'))) as epub: + titleFilePath = "OEBPS/title_page.xhtml" + contentFilePath = "OEBPS/content.opf" + + namelist = set(epub.namelist()) + #logging.debug('namelist=%s' % namelist) + if 'mimetype' not in namelist or \ + 'META-INF/container.xml' not in namelist: + #raise InvalidEPub('%s: not a valid EPUB' % filename) + logging.debug('File %s is not a valid EPub format file.' % filename) + return ret + + if contentFilePath not in namelist: + return ret # file is not newer + + data = epub.read(contentFilePath) + soup = bs.BeautifulStoneSoup(data) + lstdte = soup.find ('dc:date', {'opf:event' : 'modification'}) + #logging.debug('lstdte=%s' % lstdte.string) + if lstdte is None and titleFilePath in namelist: + data = epub.read(titleFilePath) + soup = bs.BeautifulStoneSoup(data) + fld = '' + allTDs = soup.findAll ('td') + for td in allTDs: + b = td.find ('b') + if b is not None: + fld = b.string + if td.string is not None and fld == "Updated:": + lastdate = td.string + #logging.debug('title lastdate=%s' % lastdate) + else: + lastdate = lstdte.string.strip(' ') + #logging.debug('contents lastdate=%s' % lastdate) + + if lastdate is not None: + currUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(curdte.strftime('%Y-%m-%d'), "%Y-%m-%d"))) + storyUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(lastdate, "%Y-%m-%d"))) + logging.debug('File %s last update date is %s, comparing to %s' % (filename, storyUpdated, currUpdated)) + if currUpdated <= storyUpdated : + ret = False + + logging.debug("Does %s need to be updated? %s" % (filename, ret)) + return ret + + def toZip(filename, directory): zippedHelp = zipfile.ZipFile(filename, "w", compression=zipfile.ZIP_DEFLATED) lst = os.listdir(directory)