diff --git a/adapter.py b/adapter.py index 1360824..f4dd116 100644 --- a/adapter.py +++ b/adapter.py @@ -2,6 +2,7 @@ import logging import datetime +from output import makeAcceptableFilename try: from google.appengine.api.urlfetch import fetch as googlefetch @@ -15,6 +16,18 @@ class LoginRequiredException(Exception): def __str__(self): return repr(self.url + ' requires user to be logged in') + +class StoryArchivedAlready(Exception): + pass + +class StoryDoesNotExist(Exception): + pass + +class FailedToDownload(Exception): + pass + +class InvalidStoryURL(Exception): + pass class FanfictionSiteAdapter: appEngine = appEngineGlob @@ -93,12 +106,13 @@ class FanfictionSiteAdapter: return self.uuid def getOutputName(self): - self.outputName = self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId + self.outputName = makeAcceptableFilename(self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId) logging.debug('self.outputName=%s' % self.outputName) return self.outputName - def getOutputFileName(self, booksDirectory, format): - self.outputFileName = booksDirectory + "/" + self.getOutputName() + "." + format + def getOutputFileName(self, booksDirectory, bookExt): + self.getOutputName() # make sure self.outputName is populated + self.outputFileName = booksDirectory + "/" + self.outputName + bookExt logging.debug('self.outputFileName=%s' % self.outputFileName) return self.outputFileName diff --git a/downloader.py b/downloader.py index ee0120f..3dfd7fc 100644 --- a/downloader.py +++ b/downloader.py @@ -14,12 +14,18 @@ import urlparse as up import BeautifulSoup as bs import htmlentitydefs as hdefs +import zipdir +import output +import adapter +from adapter import StoryArchivedAlready +from adapter import StoryDoesNotExist +from adapter import FailedToDownload +from adapter import InvalidStoryURL +from adapter import LoginRequiredException import ffnet import fpcom import ficwad -import output -import adapter import fictionalley import hpfiction import twilighted @@ -31,15 +37,30 @@ import time class FanficLoader: '''A controller class which handles the interaction between various specific downloaders and writers''' booksDirectory = "books" + standAlone = False - def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True): + def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True, overwrite=False): self.adapter = adapter self.writerClass = writerClass self.quiet = quiet self.inmemory = inmemory self.compress = compress self.badLogin = False - self.overWrite = True + self.overWrite = overwrite + + def getBooksDirectory(self): + return self.booksDirectory + + def setBooksDirectory(self, bd): + self.booksDirectory = bd + return self.booksDirectory + + def getStandAlone(self): + return self.standAlone + + def setStandAlone(self, sa): + self.standAlone = sa + return self.standAlone def getAdapter(): return self.adapter @@ -55,13 +76,16 @@ class FanficLoader: urls = self.adapter.extractIndividualUrls() - if (self.adapter.hasAppEngine): - self.overWrite = True + logging.debug("self.writerClass=%s" % self.writerClass) + if self.standAlone and not self.inmemory: + s = self.adapter.getOutputFileName(self.booksDirectory, self.writerClass.getFormatExt()) + logging.debug("Always overwrite? %s" % self.overWrite) + if not self.overWrite: + logging.debug("Checking if current archive of the story exists. Filename=%s" % s) + if not zipdir.checkNewer ( s, self.adapter.getStoryUpdated() ): + raise StoryArchivedAlready("A Current archive file \"" + s + "\" already exists! Skipping!") else: - s = self.adapter.getOutputFileName(self.booksDirectory, format) - if not self.overWrite and os.path.isfile(s): - print >> sys.stderr, "File " + s + " already exists! Skipping!" - exit(10) + logging.debug("Do not check for existance of archive file.") self.writer = self.writerClass(self.booksDirectory, self.adapter, inmemory=self.inmemory, compress=self.compress) @@ -83,10 +107,17 @@ class FanficLoader: if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) - (url, format) = sys.argv[1:] - # (url) = sys.argv[1] - # format = 'epub' + argvlen = len(sys.argv) + url = None + if argvlen > 1: + url = sys.argv[1] + if argvlen > 2: + bookFormat = sys.argv[2] + if url is None: + print >> sys.stderr, "Usage: downloader.py URL Type" + sys.exit(-1) + if type(url) is unicode: print('URL is unicode') url = url.encode('latin1') @@ -117,9 +148,9 @@ if __name__ == '__main__': print >> sys.stderr, "Oi! I can haz not appropriate adapter for URL %s!" % url sys.exit(1) - if format == 'epub': + if bookFormat == 'epub': writerClass = output.EPubFanficWriter - elif format == 'html': + elif bookFormat == 'html': writerClass = output.HTMLWriter if adapter.requiresLogin(url): @@ -134,5 +165,28 @@ if __name__ == '__main__': loader = FanficLoader(adapter, writerClass) - loader.download() + loader.setStandAlone(True) + + try: + loader.download() + except FailedToDownload, ftd: + print >> sys.stderr, str(ftd) + sys.exit(2) # Error Downloading + except InvalidStoryURL, isu: + print >> sys.stderr, str(isu) + sys.exit(3) # Unknown Error + except StoryArchivedAlready, se: + print >> sys.stderr, str(se) + sys.exit(10) # Skipped + except StoryDoesNotExist, sdne: + print >> sys.stderr, str(sdne) + sys.exit(20) # Missing + except LoginRequiredException, lre: + print >> sys.stderr, str(lre) + sys.exit(30) # Missing + except Exception, e: + print >> sys.stderr, str(e) + sys.exit(99) # Unknown Error + + sys.exit(0) diff --git a/ffnet.py b/ffnet.py index 7a201e2..a6fee2d 100644 --- a/ffnet.py +++ b/ffnet.py @@ -70,8 +70,7 @@ class FFNet(FanfictionSiteAdapter): logging.debug('spl=%s' % spl) if spl is not None: if len(spl) > 0 and spl[0] != 's': - logging.error("Error URL \"%s\" is not a story." % self.url) - exit (20) + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) if len(spl) > 1: self.storyId = spl[1] if len(spl) > 2: @@ -150,9 +149,23 @@ class FFNet(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = '' + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + d2 = re.sub('&\#[0-9]+;', ' ', data) - soup = bs.BeautifulStoneSoup(d2) + soup = None + try: + soup = bs.BeautifulStoneSoup(d2) + except: + logging.error("Failed to decode: <%s>" % d2) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + allA = soup.findAll('a') for a in allA: if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1: @@ -264,7 +277,15 @@ class FFNet(FanfictionSiteAdapter): def getText(self, url): time.sleep( 2.0 ) - data = self.fetchUrl(url) + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + lines = data.split('\n') textbuf = '' @@ -276,16 +297,15 @@ class FFNet(FanfictionSiteAdapter): except: data = olddata + soup = None try: soup = bs.BeautifulStoneSoup(data) except: - logging.info("Failed to decode: <%s>" % data) - soup = None + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit (20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/fictionalley.py b/fictionalley.py index fbbe95e..31f43b5 100644 --- a/fictionalley.py +++ b/fictionalley.py @@ -124,25 +124,36 @@ class FictionAlley(FanfictionSiteAdapter): def extractIndividualUrls(self): - data = self.opener.open(self.url).read() + data = '' + try: + data = self.opener.open(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") # There is some usefull information in the headers of the first chapter page.. data = data.replace('','').replace('','') - soup = bs.BeautifulStoneSoup(data) + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) breadcrumbs = soup.find('div', {'class': 'breadcrumbs'}) if breadcrumbs is not None: # Be aware that this means that the user has entered the {STORY}01.html # We will not have valid Publised and Updated dates. User should enter # the {STORY}.html instead. We should force that instead of this. - logging.debug('breadcrumbs=%s' % breadcrumbs ) + #logging.debug('breadcrumbs=%s' % breadcrumbs ) bcas = breadcrumbs.findAll('a') - logging.debug('bcas=%s' % bcas ) + #logging.debug('bcas=%s' % bcas ) if bcas is not None and len(bcas) > 1: bca = bcas[1] - logging.debug('bca=%s' % bca ) + #logging.debug('bca=%s' % bca ) if 'href' in bca._getAttrMap(): - logging.debug('bca.href=%s' % bca['href'] ) + #logging.debug('bca.href=%s' % bca['href'] ) url = str(bca['href']) if url is not None and len(url) > 0: self.url = url @@ -244,7 +255,15 @@ class FictionAlley(FanfictionSiteAdapter): def getText(self, url): # fictionalley uses full URLs in chapter list. - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + # find & and # replaced with matching div pair for easier parsing. @@ -252,13 +271,17 @@ class FictionAlley(FanfictionSiteAdapter): # something other than div prevents soup from pairing # our div with poor html inside the story text. data = data.replace('','').replace('','') - soup = bs.BeautifulStoneSoup(data) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) div = soup.find('crazytagstringnobodywouldstumbleonaccidently', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) html = soup.findAll('html') if len(html) > 1: diff --git a/ficwad.py b/ficwad.py index 7e9f3ae..a2aab78 100644 --- a/ficwad.py +++ b/ficwad.py @@ -57,9 +57,21 @@ class FicWad(FanfictionSiteAdapter): def extractIndividualUrls(self): oldurl = '' - - data = u2.urlopen(self.url).read() - soup = bs.BeautifulStoneSoup(data) + cururl = self.url + data = '' + try: + data = u2.urlopen(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) story = soup.find('div', {'id' : 'story'}) crumbtrail = story.find('h3') # the only h3 ficwad uses. @@ -100,7 +112,7 @@ class FicWad(FanfictionSiteAdapter): meta = soup.find('p', {'class' : 'meta'}) if meta is not None: s = str(meta).replace('\n',' ').replace('\t','').split(' - ') - logging.debug('meta.s=%s' % s) + #logging.debug('meta.s=%s' % s) for ss in s: s1 = ss.replace(' ','').split(':') #logging.debug('meta.s.s1=%s' % s1) @@ -164,11 +176,18 @@ class FicWad(FanfictionSiteAdapter): ii = 1 if oldurl is not None and len(oldurl) > 0: + logging.debug('Switching back to %s' % oldurl) + cururl = oldurl data = u2.urlopen(oldurl).read() soup = bs.BeautifulStoneSoup(data) storylist = soup.find('ul', {'id' : 'storylist'}) if storylist is not None: + allBlocked = storylist.findAll('li', {'class' : 'blocked'}) + if allBlocked is not None: + #logging.debug('allBlocked=%s' % allBlocked) + raise LoginRequiredException(cururl) + allH4s = storylist.findAll('h4') #logging.debug('allH4s=%s' % allH4s) @@ -216,14 +235,25 @@ class FicWad(FanfictionSiteAdapter): if url.find('http://') == -1: url = 'http://' + self.host + '/' + url - data = u2.urlopen(url).read() + data = '' + try: + data = u2.urlopen(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + try: + soup = bs.BeautifulStoneSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - soup = bs.BeautifulStoneSoup(data) div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + return div.__str__('utf8') diff --git a/fpcom.py b/fpcom.py index b943132..3598c2e 100644 --- a/fpcom.py +++ b/fpcom.py @@ -65,8 +65,7 @@ class FPCom(FanfictionSiteAdapter): spl = self.path.split('/') if spl is not None: if len(spl) > 0 and spl[0] != 's': - logging.error("Error URL \"%s\" is not a story." % self.url) - exit (20) + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) if len(spl) > 1: self.storyId = spl[1] if len(spl) > 2: @@ -138,9 +137,23 @@ class FPCom(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = '' + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + d2 = re.sub('&\#[0-9]+;', ' ', data) - soup = bs.BeautifulStoneSoup(d2) + soup = None + try: + soup = bs.BeautifulStoneSoup(d2) + except: + logging.error("Failed to decode: <%s>" % d2) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + allA = soup.findAll('a') for a in allA: if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1: @@ -277,7 +290,15 @@ class FPCom(FanfictionSiteAdapter): def getText(self, url): time.sleep( 2.0 ) - data = self.fetchUrl(url) + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + lines = data.split('\n') textbuf = '' @@ -289,16 +310,16 @@ class FPCom(FanfictionSiteAdapter): except: data = olddata + soup = None try: soup = bs.BeautifulStoneSoup(data) except: logging.info("Failed to decode: <%s>" % data) - soup = None + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'storytext'}) if None == div: - logging.error("Error downloading Chapter: %s" % url) - exit (20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/hpfiction.py b/hpfiction.py index 3850939..b2116ef 100644 --- a/hpfiction.py +++ b/hpfiction.py @@ -84,10 +84,21 @@ class HPFiction(FanfictionSiteAdapter): return self.path def extractIndividualUrls(self): + data = '' + try: + data = self.opener.open(self.url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") - data = self.opener.open(self.url).read() - soup = bs.BeautifulSoup(data) - + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + links = soup.findAll('a') def_chapurl = '' def_chaptitle = '' @@ -220,12 +231,26 @@ class HPFiction(FanfictionSiteAdapter): def getText(self, url): logging.debug('Downloading from URL: %s' % url) - data = self.opener.open(url).read() - soup = bs.BeautifulSoup(data) + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + divtext = soup.find('div', {'id' : 'fluidtext'}) if None == divtext: - logging.error("Error downloading Chapter: %s" % url) - exit(20) + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) + return divtext.__str__('utf8') diff --git a/mediaminer.py b/mediaminer.py index e8389ab..2b790f4 100644 --- a/mediaminer.py +++ b/mediaminer.py @@ -81,7 +81,9 @@ class MediaMiner(FanfictionSiteAdapter): self.storyId = ss[-2].strip() self.path = '/fanfic/view_st.php/' + self.storyId self.url = 'http://' + self.host + self.path - logging.debug('self.url=%s' % self.url) + logging.debug('self.url=%s' % self.url) + else: + raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url) logging.debug('self.storyId=%s' % self.storyId) @@ -144,9 +146,23 @@ class MediaMiner(FanfictionSiteAdapter): return True def extractIndividualUrls(self): - data = self.fetchUrl(self.url) + data = None + try: + data = self.fetchUrl(self.url) + except Exception, e: + data = None + logging.error("Caught an exception reading URL " + self.url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + self.url + "!") + #data.replace('
',' ').replace('
',' ').replace('
',' ') - soup = bs.BeautifulSoup(data) + soup = None + try: + soup = bs.BeautifulSoup(data) + except: + logging.error("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % self.url) + #logging.debug('soap=%s' % soup) urls = [] @@ -175,7 +191,7 @@ class MediaMiner(FanfictionSiteAdapter): for ii in range(ll): td = td_smtxt[ii] if 'class' in td._getAttrMap() and td['class'] != 'smtxt': - logging.debug('td has class attribute but is not smtxt') + #logging.debug('td has class attribute but is not smtxt') continue ss = str(td).replace('\n','').replace('\r','').replace(' ', ' ') #logging.debug('ss=%s' % ss) @@ -309,31 +325,28 @@ class MediaMiner(FanfictionSiteAdapter): self.numChapters = str(numchapters) logging.debug('self.numChapters=%s' % self.numChapters) - logging.debug('urls=%s' % urls) + #logging.debug('urls=%s' % urls) return urls def getText(self, url): time.sleep( 2.0 ) logging.debug('url=%s' % url) - data = self.fetchUrl(url) - + data = '' + try: + data = self.fetchUrl(url) + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None try: soup = bs.BeautifulSoup(data) except: - logging.info("Failed to decode: <%s>" % data) - soup = None - exit(20) - return '' + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - #div = soup.find('div', {'id' : 'storytext'}) - #if div is None: - #logging.error("Error downloading Chapter: %s" % url) - #exit (20) - #return '' - - #logging.info("Soup: %s" % soup.prettify()) - nvs = bs.NavigableString('') sst='' allAs = soup.findAll ('a', { 'name' : 'fic_c' }) @@ -368,16 +381,9 @@ class MediaMiner(FanfictionSiteAdapter): sst = sst + st nxta = nxta.nextSibling - #sst = sst.replace(' ',' ').strip() - #logging.debug('sst=%s' % sst) + if sst is None: + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) - #logging.debug('sst.0=%s' % sst) - #sst0 = sst.replace(u'≴', u'“').replace(u'≵','”').replace(u'≰',u'‘').replace(u'≱',u'’') - #sst0 = sst.replace(u"≵","”") - #logging.debug('sst.1=%s' % sst0) - #sst1 = sst.replace(u'≴', u'\"').replace('≵','\"').replace('≰','\'').replace('≱','\'') - #logging.debug('sst.2=%s' % sst1) - return sst class FPC_UnitTests(unittest.TestCase): diff --git a/output.py b/output.py index a320a42..64a64d0 100644 --- a/output.py +++ b/output.py @@ -40,9 +40,25 @@ class FanficWriter: def finalise(self): pass + @staticmethod + def getFormatName(): + return 'base' + + @staticmethod + def getFormatExt(): + return '.bse' + class TextWriter(FanficWriter): htmlWriter = None + @staticmethod + def getFormatName(): + return 'text' + + @staticmethod + def getFormatExt(): + return '.txt' + def __init__(self, base, adapter, inmemory=False, compress=False): self.htmlWriter = HTMLWriter(base, adapter, True, False) @@ -59,11 +75,19 @@ class TextWriter(FanficWriter): class HTMLWriter(FanficWriter): body = '' + @staticmethod + def getFormatName(): + return 'html' + + @staticmethod + def getFormatExt(): + return '.html' + def __init__(self, base, adapter, inmemory=False, compress=False): self.basePath = base self.storyTitle = removeEntities(adapter.getStoryName()) self.name = makeAcceptableFilename(adapter.getOutputName()) - self.fileName = self.basePath + '/' + self.name + '.html' + self.fileName = self.basePath + '/' + self.name + self.getFormatExt() self.authorName = removeEntities(adapter.getAuthorName()) self.adapter = adapter @@ -111,6 +135,45 @@ class EPubFanficWriter(FanficWriter): files = {} + @staticmethod + def getFormatName(): + return 'epub' + + @staticmethod + def getFormatExt(): + return '.epub' + + def __init__(self, base, adapter, inmemory=False, compress=True): + self.basePath = base + self.storyTitle = removeEntities(adapter.getStoryName()) + self.name = makeAcceptableFilename(adapter.getOutputName()) + self.directory = self.basePath + '/' + self.name + self.authorName = removeEntities(adapter.getAuthorName()) + self.inmemory = inmemory + self.adapter = adapter + + self.files = {} + self.chapters = [] + + if not self.inmemory: + self.inmemory = True + self.writeToFile = True + else: + self.writeToFile = False + + if not self.inmemory: + if os.path.exists(self.directory): + shutil.rmtree(self.directory) + + os.mkdir(self.directory) + + os.mkdir(self.directory + '/META-INF') + os.mkdir(self.directory + '/OEBPS') + + self._writeFile('mimetype', MIMETYPE) + self._writeFile('META-INF/container.xml', CONTAINER) + self._writeFile('OEBPS/stylesheet.css', CSS) + def _writeFile(self, fileName, data): #logging.debug('_writeFile(`%s`, data)' % fileName) if fileName in self.files: @@ -134,39 +197,6 @@ class EPubFanficWriter(FanficWriter): for f in self.files: self.files[f].close() - def __init__(self, base, adapter, inmemory=False, compress=True): - self.basePath = base - self.storyTitle = removeEntities(adapter.getStoryName()) - self.name = makeAcceptableFilename(adapter.getOutputName()) - self.directory = self.basePath + '/' + self.name - self.authorName = removeEntities(adapter.getAuthorName()) - self.inmemory = inmemory - self.adapter = adapter - - self.files = {} - self.chapters = [] - - if not self.inmemory: - self.inmemory = True - self.writeToFile = True - else: - self.writeToFile = False - - - - if not self.inmemory: - if os.path.exists(self.directory): - shutil.rmtree(self.directory) - - os.mkdir(self.directory) - - os.mkdir(self.directory + '/META-INF') - os.mkdir(self.directory + '/OEBPS') - - self._writeFile('mimetype', MIMETYPE) - self._writeFile('META-INF/container.xml', CONTAINER) - self._writeFile('OEBPS/stylesheet.css', CSS) - def writeChapter(self, index, title, text): title = removeEntities(title) logging.debug("Writing chapter: %s" % title) @@ -321,7 +351,7 @@ class EPubFanficWriter(FanficWriter): self._closeFiles() - filename = self.directory + '.epub' + filename = self.directory + self.getFormatExt() zipdata = zipdir.inMemoryZip(self.files) @@ -382,4 +412,4 @@ def removeEntities(text): return text def makeAcceptableFilename(text): - return re.sub('[^a-zA-Z0-9_\'-]+','',removeEntities(text).replace(" ", "_").replace(":","_")) + return re.sub('[^a-zA-Z0-9_-]+','',removeEntities(text).replace(" ", "_").replace(":","_")) diff --git a/potionsNsnitches.py b/potionsNsnitches.py index 3868e93..d0c3fa5 100644 --- a/potionsNsnitches.py +++ b/potionsNsnitches.py @@ -130,15 +130,35 @@ class PotionsNSnitches(FanfictionSiteAdapter): def extractIndividualUrls(self): url = self.url + '&chapter=1' - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") if self.reqLoginData(data): - self.performLogin() - data = self.opener.open(url).read() - if self.reqLoginData(data): - return None + self.performLogin() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") - soup = bs.BeautifulStoneSoup(data) + if self.reqLoginData(data): + raise FailedToDownload("Error downloading Story: %s! Login Failed!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % url) self.storyName = '' self.authorName = '' @@ -302,16 +322,29 @@ class PotionsNSnitches(FanfictionSiteAdapter): logging.debug('Getting data from: %s' % url) - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) # need to do this, because for some reason the
tag in the story causes problems data = data.replace('
', ' SOMETHING_BR ') - soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) - + + soup = None + try: + soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) + div = soup.find('div', {'id' : 'story'}) if None == div: - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) # put the
tags back in.. text = div.__str__('utf8').replace(' SOMETHING_BR ','
') diff --git a/twilighted.py b/twilighted.py index ed29340..00f5936 100644 --- a/twilighted.py +++ b/twilighted.py @@ -110,15 +110,36 @@ class Twilighted(FanfictionSiteAdapter): def extractIndividualUrls(self): url = self.url + '&chapter=1' - data = self.opener.open(url).read() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") if self.reqLoginData(data): - self.performLogin() - data = self.opener.open(url).read() - if self.reqLoginData(data): - return None + self.performLogin() + + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise StoryDoesNotExist("Problem reading story URL " + url + "!") - soup = bs.BeautifulStoneSoup(data) + if self.reqLoginData(data): + raise FailedToDownload("Error downloading Story: %s! Login Failed!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data) + except: + raise FailedToDownload("Error downloading Story: %s! Problem decoding page!" % url) title = soup.find('title').string logging.debug('Title: %s' % title) @@ -254,14 +275,26 @@ class Twilighted(FanfictionSiteAdapter): logging.debug('Getting data from: %s' % url) - data = self.opener.open(url).read() + data = '' + try: + data = self.opener.open(url).read() + except Exception, e: + data = '' + logging.error("Caught an exception reading URL " + url + ". Exception " + str(e) + ".") + if data is None: + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) + + soup = None + try: + soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) + except: + logging.info("Failed to decode: <%s>" % data) + raise FailedToDownload("Error downloading Chapter: %s! Problem decoding page!" % url) - soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES) - div = soup.find('div', {'id' : 'story'}) if None == div: - return '' + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8') diff --git a/zipdir.py b/zipdir.py index 597b4f8..6079e08 100644 --- a/zipdir.py +++ b/zipdir.py @@ -1,11 +1,79 @@ # -*- coding: utf-8 -*- +import sys import os +import zlib import zipfile +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED +from contextlib import closing import logging +import BeautifulSoup as bs +import htmlentitydefs as hdefs +import time +import datetime +from datetime import timedelta + import StringIO +class InvalidEPub(Exception): + pass + +def checkNewer(filename, curdte): + ret = True + + if not os.path.isfile(filename): + logging.debug('File %s does not already exist.' % filename) + return ret + + #logging.debug('filename=%s, curdte=%s' % (filename, curdte)) + lastdate = None + with closing(ZipFile(open(filename, 'rb'))) as epub: + titleFilePath = "OEBPS/title_page.xhtml" + contentFilePath = "OEBPS/content.opf" + + namelist = set(epub.namelist()) + #logging.debug('namelist=%s' % namelist) + if 'mimetype' not in namelist or \ + 'META-INF/container.xml' not in namelist: + #raise InvalidEPub('%s: not a valid EPUB' % filename) + logging.debug('File %s is not a valid EPub format file.' % filename) + return ret + + if contentFilePath not in namelist: + return ret # file is not newer + + data = epub.read(contentFilePath) + soup = bs.BeautifulStoneSoup(data) + lstdte = soup.find ('dc:date', {'opf:event' : 'modification'}) + #logging.debug('lstdte=%s' % lstdte.string) + if lstdte is None and titleFilePath in namelist: + data = epub.read(titleFilePath) + soup = bs.BeautifulStoneSoup(data) + fld = '' + allTDs = soup.findAll ('td') + for td in allTDs: + b = td.find ('b') + if b is not None: + fld = b.string + if td.string is not None and fld == "Updated:": + lastdate = td.string + #logging.debug('title lastdate=%s' % lastdate) + else: + lastdate = lstdte.string.strip(' ') + #logging.debug('contents lastdate=%s' % lastdate) + + if lastdate is not None: + currUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(curdte.strftime('%Y-%m-%d'), "%Y-%m-%d"))) + storyUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(lastdate, "%Y-%m-%d"))) + logging.debug('File %s last update date is %s, comparing to %s' % (filename, storyUpdated, currUpdated)) + if currUpdated <= storyUpdated : + ret = False + + logging.debug("Does %s need to be updated? %s" % (filename, ret)) + return ret + + def toZip(filename, directory): zippedHelp = zipfile.ZipFile(filename, "w", compression=zipfile.ZIP_DEFLATED) lst = os.listdir(directory)