Raise exceptions instead of an exit when errors are detected.

Added static functions to the writer classes that contain the type name and type extension to use when writing out the destination file.
Added a function to the zipdir that will check the date the fic was last updated in the current archive of a fic against the current last updated date and return whether the fic is newer then the archive file.  This is not useful at this time for when this is running under appEngine.
Added a flag to indicate if the program is running from appEngine or not.

adapter.py

@@ -2,6 +2,7 @@
 
 import logging
 import datetime
+from output import makeAcceptableFilename
 
 try:
 	from google.appengine.api.urlfetch import fetch as googlefetch
@@ -15,6 +16,18 @@ class LoginRequiredException(Exception):
 	
 	def __str__(self):
 		return repr(self.url + ' requires user to be logged in')
+	
+class StoryArchivedAlready(Exception):
+	pass
+
+class StoryDoesNotExist(Exception):
+	pass
+
+class FailedToDownload(Exception):
+	pass
+
+class InvalidStoryURL(Exception):
+	pass
 
 class FanfictionSiteAdapter:
 	appEngine = appEngineGlob
@@ -93,12 +106,13 @@ class FanfictionSiteAdapter:
 		return self.uuid
 
 	def getOutputName(self):
-		self.outputName = self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId
+		self.outputName = makeAcceptableFilename(self.storyName.replace(" ", "_") + self.outputStorySep + self.storyId)
 		logging.debug('self.outputName=%s' % self.outputName)
 		return self.outputName
 
-	def getOutputFileName(self, booksDirectory, format):
-		self.outputFileName = booksDirectory + "/" + self.getOutputName() + "." + format
+	def getOutputFileName(self, booksDirectory, bookExt):
+		self.getOutputName()	# make sure self.outputName is populated
+		self.outputFileName = booksDirectory + "/" + self.outputName + bookExt
 		logging.debug('self.outputFileName=%s' % self.outputFileName)
 		return self.outputFileName
 

downloader.py

@@ -14,12 +14,18 @@ import urlparse as up
 import BeautifulSoup as bs
 import htmlentitydefs as hdefs
 
+import zipdir
 
+import output
+import adapter
+from adapter import StoryArchivedAlready
+from adapter import StoryDoesNotExist
+from adapter import FailedToDownload
+from adapter import InvalidStoryURL
+from adapter import LoginRequiredException
 import ffnet
 import fpcom
 import ficwad
-import output
-import adapter
 import fictionalley
 import hpfiction
 import twilighted
@@ -31,15 +37,30 @@ import time
 class FanficLoader:
 	'''A controller class which handles the interaction between various specific downloaders and writers'''
 	booksDirectory = "books"
+	standAlone = False
 	
-	def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True):
+	def __init__(self, adapter, writerClass, quiet = False, inmemory = False, compress=True, overwrite=False):
 		self.adapter = adapter
 		self.writerClass = writerClass
 		self.quiet = quiet
 		self.inmemory = inmemory
 		self.compress = compress
 		self.badLogin = False
-		self.overWrite = True
+		self.overWrite = overwrite
+			
+	def getBooksDirectory(self):
+		return self.booksDirectory
+
+	def setBooksDirectory(self, bd):
+		self.booksDirectory = bd
+		return self.booksDirectory
+	
+	def getStandAlone(self):
+		return self.standAlone
+
+	def setStandAlone(self, sa):
+		self.standAlone = sa
+		return self.standAlone
 	
 	def getAdapter():
 		return self.adapter
@@ -55,13 +76,16 @@ class FanficLoader:
 		
 		urls = self.adapter.extractIndividualUrls()
 
-		if (self.adapter.hasAppEngine):
-			self.overWrite = True
+		logging.debug("self.writerClass=%s" % self.writerClass)
+		if self.standAlone and not self.inmemory:
+			s = self.adapter.getOutputFileName(self.booksDirectory, self.writerClass.getFormatExt())
+			logging.debug("Always overwrite? %s" % self.overWrite)
+			if not self.overWrite:
+				logging.debug("Checking if current archive of the story exists.  Filename=%s" % s)
+				if not zipdir.checkNewer ( s, self.adapter.getStoryUpdated() ):
+					raise StoryArchivedAlready("A Current archive file \"" + s + "\" already exists!  Skipping!")
 		else:
-			s = self.adapter.getOutputFileName(self.booksDirectory, format)
-			if not self.overWrite and os.path.isfile(s):
-				print >> sys.stderr, "File " + s + " already exists!  Skipping!"
-				exit(10)
+			logging.debug("Do not check for existance of archive file.")
 
 		self.writer = self.writerClass(self.booksDirectory, self.adapter, inmemory=self.inmemory, compress=self.compress)
 		
@@ -83,10 +107,17 @@ class FanficLoader:
 
 if __name__ == '__main__':
 	logging.basicConfig(level=logging.DEBUG)
-	(url, format) = sys.argv[1:]
-	# (url) = sys.argv[1]
-	# format = 'epub'
+	argvlen = len(sys.argv)
+	url = None
+	if argvlen > 1:
+		url = sys.argv[1]
+	if argvlen > 2:
+		bookFormat = sys.argv[2]
 	
+	if url is None: 
+		print >> sys.stderr, "Usage: downloader.py URL Type"
+		sys.exit(-1)
+		
 	if type(url) is unicode:
 		print('URL is unicode')
 		url = url.encode('latin1')
@@ -117,9 +148,9 @@ if __name__ == '__main__':
 		print >> sys.stderr, "Oi! I can haz not appropriate adapter for URL %s!" % url
 		sys.exit(1)
 
-	if format == 'epub':
+	if bookFormat == 'epub':
 		writerClass = output.EPubFanficWriter
-	elif format == 'html':
+	elif bookFormat == 'html':
 		writerClass = output.HTMLWriter
 	
 	if adapter.requiresLogin(url):
@@ -134,5 +165,28 @@ if __name__ == '__main__':
 		
 	
 	loader = FanficLoader(adapter, writerClass)
-	loader.download()
+	loader.setStandAlone(True)
+
+	try:
+		loader.download()
+	except FailedToDownload, ftd:
+		print >> sys.stderr, str(ftd)
+		sys.exit(2)		# Error Downloading
+	except InvalidStoryURL, isu:
+		print >> sys.stderr, str(isu)
+		sys.exit(3)		# Unknown Error
+	except StoryArchivedAlready, se:
+		print >> sys.stderr, str(se)
+		sys.exit(10)	# Skipped
+	except StoryDoesNotExist, sdne:
+		print >> sys.stderr, str(sdne)
+		sys.exit(20) 	# Missing
+	except LoginRequiredException, lre:
+		print >> sys.stderr, str(lre)
+		sys.exit(30) 	# Missing
+	except Exception, e:
+		print >> sys.stderr, str(e)
+		sys.exit(99)		# Unknown Error
+	
+	sys.exit(0)
 	

ffnet.py

@@ -70,8 +70,7 @@ class FFNet(FanfictionSiteAdapter):
 		logging.debug('spl=%s' % spl)
 		if spl is not None:
 			if len(spl) > 0 and spl[0] != 's':
-				logging.error("Error URL \"%s\" is not a story." % self.url)
-				exit (20)				
+				raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url)
 			if len(spl) > 1:
 				self.storyId = spl[1]
 			if len(spl) > 2:
@@ -150,9 +149,23 @@ class FFNet(FanfictionSiteAdapter):
 		return True
 
 	def extractIndividualUrls(self):
-		data = self.fetchUrl(self.url)
+		data = ''
+		try:
+			data = self.fetchUrl(self.url)
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
+		
 		d2 = re.sub('&\#[0-9]+;', ' ', data)
-		soup = bs.BeautifulStoneSoup(d2)
+		soup = None
+		try:
+			soup = bs.BeautifulStoneSoup(d2)
+		except:
+			logging.error("Failed to decode: <%s>" % d2)
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
+
 		allA = soup.findAll('a')
 		for a in allA:
 			if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1:
@@ -264,7 +277,15 @@ class FFNet(FanfictionSiteAdapter):
 	
 	def getText(self, url):
 		time.sleep( 2.0 )
-		data = self.fetchUrl(url)
+		data = ''
+		try:
+			data = self.fetchUrl(url)
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
 		lines = data.split('\n')
 		
 		textbuf = ''
@@ -276,16 +297,15 @@ class FFNet(FanfictionSiteAdapter):
 		except:
 			data = olddata
 		
+		soup = None
 		try:
 			soup = bs.BeautifulStoneSoup(data)
 		except:
-			logging.info("Failed to decode: <%s>" % data)
-			soup = None
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
+
 		div = soup.find('div', {'id' : 'storytext'})
 		if None == div:
-			logging.error("Error downloading Chapter: %s" % url)
-			exit (20)
-			return '<html/>'
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 			
 		return div.__str__('utf8')
 					

fictionalley.py

@@ -124,25 +124,36 @@ class FictionAlley(FanfictionSiteAdapter):
 	
 		
 	def extractIndividualUrls(self):
-		data = self.opener.open(self.url).read()		
+		data = ''
+		try:
+			data = self.opener.open(self.url).read()		
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
 		
 		# There is some usefull information in the headers of the first chapter page..
 		data = data.replace('<!-- headerstart -->','<crazytagstringnobodywouldstumbleonaccidently id="storyheaders">').replace('<!-- headerend -->','</crazytagstringnobodywouldstumbleonaccidently>')
-		soup = bs.BeautifulStoneSoup(data)
+		soup = None
+		try:
+			soup = bs.BeautifulStoneSoup(data)
+		except:
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
 				
 		breadcrumbs = soup.find('div', {'class': 'breadcrumbs'})
 		if breadcrumbs is not None:
 			# Be aware that this means that the user has entered the {STORY}01.html 
 			# We will not have valid Publised and Updated dates.  User should enter 
 			# the {STORY}.html instead.  We should force that instead of this.
-			logging.debug('breadcrumbs=%s' % breadcrumbs )
+			#logging.debug('breadcrumbs=%s' % breadcrumbs )
 			bcas = breadcrumbs.findAll('a')
-			logging.debug('bcas=%s' % bcas )
+			#logging.debug('bcas=%s' % bcas )
 			if bcas is not None and len(bcas) > 1:
 				bca = bcas[1]
-				logging.debug('bca=%s' % bca )
+				#logging.debug('bca=%s' % bca )
 				if 'href' in bca._getAttrMap():
-					logging.debug('bca.href=%s' % bca['href'] )
+					#logging.debug('bca.href=%s' % bca['href'] )
 					url = str(bca['href'])
 					if url is not None and len(url) > 0:
 						self.url = url
@@ -244,7 +255,15 @@ class FictionAlley(FanfictionSiteAdapter):
 	
 	def getText(self, url):
 		# fictionalley uses full URLs in chapter list.
-		data = self.opener.open(url).read()
+		data = ''
+		try:
+			data = self.opener.open(url).read()
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
 		
 		# find <!-- headerend --> & <!-- footerstart --> and
 		# replaced with matching div pair for easier parsing.
@@ -252,13 +271,17 @@ class FictionAlley(FanfictionSiteAdapter):
 		# something other than div prevents soup from pairing
 		# our div with poor html inside the story text.
 		data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
-		soup = bs.BeautifulStoneSoup(data)
+		
+		soup = None
+		try:
+			soup = bs.BeautifulStoneSoup(data)
+		except:
+			logging.info("Failed to decode: <%s>" % data)
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
 		
 		div = soup.find('crazytagstringnobodywouldstumbleonaccidently', {'id' : 'storytext'})
 		if None == div:
-			logging.error("Error downloading Chapter: %s" % url)
-			exit(20)
-			return '<html/>'
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
 		html = soup.findAll('html')
 		if len(html) > 1:

ficwad.py

@@ -57,9 +57,21 @@ class FicWad(FanfictionSiteAdapter):
 
 	def extractIndividualUrls(self):
 		oldurl = ''
-		
-		data = u2.urlopen(self.url).read()
-		soup = bs.BeautifulStoneSoup(data)
+		cururl = self.url
+		data = ''
+		try:
+			data = u2.urlopen(self.url).read()
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
+
+		soup = None
+		try:
+			soup = bs.BeautifulStoneSoup(data)
+		except:
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
 		
 		story = soup.find('div', {'id' : 'story'})
 		crumbtrail = story.find('h3') # the only h3 ficwad uses.
@@ -100,7 +112,7 @@ class FicWad(FanfictionSiteAdapter):
 		meta = soup.find('p', {'class' : 'meta'})
 		if meta is not None:
 			s = str(meta).replace('\n',' ').replace('\t','').split(' - ')
-			logging.debug('meta.s=%s' % s)
+			#logging.debug('meta.s=%s' % s)
 			for ss in s:
 				s1 = ss.replace(' ','').split(':')
 				#logging.debug('meta.s.s1=%s' % s1)
@@ -164,11 +176,18 @@ class FicWad(FanfictionSiteAdapter):
 		ii = 1
 
 		if oldurl is not None and len(oldurl) > 0:
+			logging.debug('Switching back to %s' % oldurl)
+			cururl = oldurl
 			data = u2.urlopen(oldurl).read()
 			soup = bs.BeautifulStoneSoup(data)
 			
 		storylist = soup.find('ul', {'id' : 'storylist'})
 		if storylist is not None:
+			allBlocked = storylist.findAll('li', {'class' : 'blocked'})
+			if allBlocked is not None:
+				#logging.debug('allBlocked=%s' % allBlocked)
+				raise LoginRequiredException(cururl)
+
 			allH4s = storylist.findAll('h4')
 			#logging.debug('allH4s=%s' % allH4s)
 	
@@ -216,14 +235,25 @@ class FicWad(FanfictionSiteAdapter):
 		if url.find('http://') == -1:
 			url = 'http://' + self.host + '/' + url
 		
-		data = u2.urlopen(url).read()
+		data = ''
+		try:
+			data = u2.urlopen(url).read()
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
+		try:
+			soup = bs.BeautifulStoneSoup(data)
+		except:
+			logging.info("Failed to decode: <%s>" % data)
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
 		
-		soup = bs.BeautifulStoneSoup(data)
 		div = soup.find('div', {'id' : 'storytext'})
 		if None == div:
-			logging.error("Error downloading Chapter: %s" % url)
-			exit(20)
-			return '<html/>'
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
 		return div.__str__('utf8')
 	
 		

fpcom.py

@@ -65,8 +65,7 @@ class FPCom(FanfictionSiteAdapter):
 		spl = self.path.split('/')
 		if spl is not None:
 			if len(spl) > 0 and spl[0] != 's':
-				logging.error("Error URL \"%s\" is not a story." % self.url)
-				exit (20)				
+				raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url)
 			if len(spl) > 1:
 				self.storyId = spl[1]
 			if len(spl) > 2:
@@ -138,9 +137,23 @@ class FPCom(FanfictionSiteAdapter):
 		return True
 	
 	def extractIndividualUrls(self):
-		data = self.fetchUrl(self.url)
+		data = ''
+		try:
+			data = self.fetchUrl(self.url)
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
+		
 		d2 = re.sub('&\#[0-9]+;', ' ', data)
-		soup = bs.BeautifulStoneSoup(d2)
+		soup = None
+		try:
+			soup = bs.BeautifulStoneSoup(d2)
+		except:
+			logging.error("Failed to decode: <%s>" % d2)
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
+
 		allA = soup.findAll('a')
 		for a in allA:
 			if 'href' in a._getAttrMap() and a['href'].find('/u/') != -1:
@@ -277,7 +290,15 @@ class FPCom(FanfictionSiteAdapter):
 	
 	def getText(self, url):
 		time.sleep( 2.0 )
-		data = self.fetchUrl(url)
+		data = ''
+		try:
+			data = self.fetchUrl(url)
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
 		lines = data.split('\n')
 		
 		textbuf = ''
@@ -289,16 +310,16 @@ class FPCom(FanfictionSiteAdapter):
 		except:
 			data = olddata
 		
+		soup = None
 		try:
 			soup = bs.BeautifulStoneSoup(data)
 		except:
 			logging.info("Failed to decode: <%s>" % data)
-			soup = None
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
+
 		div = soup.find('div', {'id' : 'storytext'})
 		if None == div:
-			logging.error("Error downloading Chapter: %s" % url)
-			exit (20)
-			return '<html/>'
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 			
 		return div.__str__('utf8')
 					

hpfiction.py

@@ -84,10 +84,21 @@ class HPFiction(FanfictionSiteAdapter):
 		return self.path
 	
 	def extractIndividualUrls(self):
+		data = ''
+		try:
+			data = self.opener.open(self.url).read()
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
 		
-		data = self.opener.open(self.url).read()
-		soup = bs.BeautifulSoup(data)
-		
+		soup = None
+		try:
+			soup = bs.BeautifulSoup(data)
+		except:
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
+				
 		links = soup.findAll('a')
 		def_chapurl = ''
 		def_chaptitle = ''
@@ -220,12 +231,26 @@ class HPFiction(FanfictionSiteAdapter):
 
 	def getText(self, url):
 		logging.debug('Downloading from URL: %s' % url)
-		data = self.opener.open(url).read()
-		soup = bs.BeautifulSoup(data)
+		data = ''
+		try:
+			data = self.opener.open(url).read()
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
+		soup = None
+		try:
+			soup = bs.BeautifulSoup(data)
+		except:
+			logging.info("Failed to decode: <%s>" % data)
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
+		
 		divtext = soup.find('div', {'id' : 'fluidtext'})
 		if None == divtext:
-			logging.error("Error downloading Chapter: %s" % url)
-			exit(20)
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
 		return divtext.__str__('utf8')
 
 

mediaminer.py

@@ -81,7 +81,9 @@ class MediaMiner(FanfictionSiteAdapter):
 				self.storyId = ss[-2].strip()
 				self.path = '/fanfic/view_st.php/' + self.storyId
 				self.url = 'http://' + self.host + self.path
-				logging.debug('self.url=%s' % self.url)			
+				logging.debug('self.url=%s' % self.url)
+		else:			
+			raise InvalidStoryURL("Error URL \"%s\" is not a story." % self.url)
 			
 		logging.debug('self.storyId=%s' % self.storyId)
 		
@@ -144,9 +146,23 @@ class MediaMiner(FanfictionSiteAdapter):
 		return True
 
 	def extractIndividualUrls(self):
-		data = self.fetchUrl(self.url)
+		data = None
+		try:
+			data = self.fetchUrl(self.url)
+		except Exception, e:
+			data = None
+			logging.error("Caught an exception reading URL " + self.url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise StoryDoesNotExist("Problem reading story URL " + self.url + "!")
+		
 		#data.replace('<br />',' ').replace('<br>',' ').replace('</br>',' ')
-		soup = bs.BeautifulSoup(data)
+		soup = None
+		try:
+			soup = bs.BeautifulSoup(data)
+		except:
+			logging.error("Failed to decode: <%s>" % data)
+			raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % self.url)
+
 		#logging.debug('soap=%s' % soup)
 		urls = []
 		
@@ -175,7 +191,7 @@ class MediaMiner(FanfictionSiteAdapter):
 			for ii in range(ll):
 				td = td_smtxt[ii]
 				if 'class' in td._getAttrMap() and td['class'] != 'smtxt':
-					logging.debug('td has class attribute but is not smtxt')
+					#logging.debug('td has class attribute but is not smtxt')
 					continue
 				ss = str(td).replace('\n','').replace('\r','').replace('&nbsp;', ' ')
 				#logging.debug('ss=%s' % ss)
@@ -309,31 +325,28 @@ class MediaMiner(FanfictionSiteAdapter):
 			
 		self.numChapters = str(numchapters)
 		logging.debug('self.numChapters=%s' % self.numChapters)
-		logging.debug('urls=%s' % urls)
+		#logging.debug('urls=%s' % urls)
 		
 		return urls
 	
 	def getText(self, url):
 		time.sleep( 2.0 )
 		logging.debug('url=%s' % url)
-		data = self.fetchUrl(url)
-
+		data = ''
+		try:
+			data = self.fetchUrl(url)
+		except Exception, e:
+			data = ''
+			logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+		if data is None:
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+		
+		soup = None
 		try:
 			soup = bs.BeautifulSoup(data)
 		except:
-			logging.info("Failed to decode: <%s>" % data)
-			soup = None
-			exit(20)
-			return '<html/>'
+			raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
 		
-		#div = soup.find('div', {'id' : 'storytext'})
-		#if div is None:
-			#logging.error("Error downloading Chapter: %s" % url)
-			#exit (20)
-			#return '<html/>'
-			
-		#logging.info("Soup: %s" % soup.prettify())
-
 		nvs = bs.NavigableString('')
 		sst=''
 		allAs = soup.findAll ('a', { 'name' : 'fic_c' })
@@ -368,16 +381,9 @@ class MediaMiner(FanfictionSiteAdapter):
 				sst = sst + st
 				nxta = nxta.nextSibling
 
-		#sst = sst.replace('&nbsp;',' ').strip()
-		#logging.debug('sst=%s' % sst)
+		if sst is None:	
+			raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 		
-		#logging.debug('sst.0=%s' % sst)
-		#sst0 = sst.replace(u'&#8820;', u'&ldquo;').replace(u'&#8821;','&rdquo;').replace(u'&#8816;',u'&lsquo;').replace(u'&#8817;',u'&rsquo;')
-		#sst0 = sst.replace(u"&#8821;","&rdquo;")
-		#logging.debug('sst.1=%s' % sst0)
-		#sst1 = sst.replace(u'&#8820;', u'\"').replace('&#8821;','\"').replace('&#8816;','\'').replace('&#8817;','\'')
-		#logging.debug('sst.2=%s' % sst1)
-					
 		return sst
 			
 class FPC_UnitTests(unittest.TestCase):

output.py

@@ -40,9 +40,25 @@ class FanficWriter:
 	def finalise(self):
 		pass
 
+	@staticmethod
+	def getFormatName():
+		return 'base'
+	
+	@staticmethod	
+	def getFormatExt():
+		return '.bse'
+	
 class TextWriter(FanficWriter):
 	htmlWriter = None
 	
+	@staticmethod
+	def getFormatName():
+		return 'text'
+	
+	@staticmethod	
+	def getFormatExt():
+		return '.txt'
+	
 	def __init__(self, base, adapter, inmemory=False, compress=False):
 		self.htmlWriter = HTMLWriter(base, adapter, True, False)
 	
@@ -59,11 +75,19 @@ class TextWriter(FanficWriter):
 class HTMLWriter(FanficWriter):
 	body = ''
 	
+	@staticmethod
+	def getFormatName():
+		return 'html'
+	
+	@staticmethod	
+	def getFormatExt():
+		return '.html'
+	
 	def __init__(self, base, adapter, inmemory=False, compress=False):
 		self.basePath = base
 		self.storyTitle = removeEntities(adapter.getStoryName())
 		self.name = makeAcceptableFilename(adapter.getOutputName())
-		self.fileName = self.basePath + '/' + self.name + '.html'
+		self.fileName = self.basePath + '/' + self.name + self.getFormatExt()
 		self.authorName = removeEntities(adapter.getAuthorName())
 		self.adapter = adapter
 		
@@ -111,6 +135,45 @@ class EPubFanficWriter(FanficWriter):
 	
 	files = {}
 	
+	@staticmethod
+	def getFormatName():
+		return 'epub'
+	
+	@staticmethod	
+	def getFormatExt():
+		return '.epub'
+	
+	def __init__(self, base, adapter, inmemory=False, compress=True):
+		self.basePath = base
+		self.storyTitle = removeEntities(adapter.getStoryName())
+		self.name = makeAcceptableFilename(adapter.getOutputName())
+		self.directory = self.basePath + '/' + self.name
+		self.authorName = removeEntities(adapter.getAuthorName())
+		self.inmemory = inmemory
+		self.adapter = adapter
+		
+		self.files = {}
+		self.chapters = []
+		
+		if not self.inmemory:
+			self.inmemory = True
+			self.writeToFile = True
+		else:
+			self.writeToFile = False
+
+		if not self.inmemory:
+			if os.path.exists(self.directory):
+				shutil.rmtree(self.directory)
+		
+			os.mkdir(self.directory)
+		
+			os.mkdir(self.directory + '/META-INF')
+			os.mkdir(self.directory + '/OEBPS')
+		
+		self._writeFile('mimetype', MIMETYPE)
+		self._writeFile('META-INF/container.xml', CONTAINER)
+		self._writeFile('OEBPS/stylesheet.css', CSS)
+
 	def _writeFile(self, fileName, data):
 		#logging.debug('_writeFile(`%s`, data)' % fileName)
 		if fileName in self.files:
@@ -134,39 +197,6 @@ class EPubFanficWriter(FanficWriter):
 			for f in self.files:
 				self.files[f].close()
 	
-	def __init__(self, base, adapter, inmemory=False, compress=True):
-		self.basePath = base
-		self.storyTitle = removeEntities(adapter.getStoryName())
-		self.name = makeAcceptableFilename(adapter.getOutputName())
-		self.directory = self.basePath + '/' + self.name
-		self.authorName = removeEntities(adapter.getAuthorName())
-		self.inmemory = inmemory
-		self.adapter = adapter
-		
-		self.files = {}
-		self.chapters = []
-		
-		if not self.inmemory:
-			self.inmemory = True
-			self.writeToFile = True
-		else:
-			self.writeToFile = False
-		
-
-
-		if not self.inmemory:
-			if os.path.exists(self.directory):
-				shutil.rmtree(self.directory)
-		
-			os.mkdir(self.directory)
-		
-			os.mkdir(self.directory + '/META-INF')
-			os.mkdir(self.directory + '/OEBPS')
-		
-		self._writeFile('mimetype', MIMETYPE)
-		self._writeFile('META-INF/container.xml', CONTAINER)
-		self._writeFile('OEBPS/stylesheet.css', CSS)
-
 	def writeChapter(self, index, title, text):
 		title = removeEntities(title)
 		logging.debug("Writing chapter: %s" % title)
@@ -321,7 +351,7 @@ class EPubFanficWriter(FanficWriter):
 		
 		self._closeFiles()
 		
-		filename = self.directory + '.epub'
+		filename = self.directory + self.getFormatExt()
 		
 		zipdata = zipdir.inMemoryZip(self.files)
 		
@@ -382,4 +412,4 @@ def removeEntities(text):
 	return text
 	
 def makeAcceptableFilename(text):
-	return re.sub('[^a-zA-Z0-9_\'-]+','',removeEntities(text).replace(" ", "_").replace(":","_"))	
+	return re.sub('[^a-zA-Z0-9_-]+','',removeEntities(text).replace(" ", "_").replace(":","_"))	

potionsNsnitches.py

@@ -130,15 +130,35 @@ class PotionsNSnitches(FanfictionSiteAdapter):
 
     def extractIndividualUrls(self):
         url = self.url + '&chapter=1'
-        data = self.opener.open(url).read()
+        data = ''
+        try:
+            data = self.opener.open(url).read()
+        except Exception, e:
+            data = ''
+            logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+        if data is None:
+            raise StoryDoesNotExist("Problem reading story URL " + url + "!")
         
         if self.reqLoginData(data):
-          self.performLogin()
-          data = self.opener.open(url).read()
-          if self.reqLoginData(data):
-            return None
+            self.performLogin()
+            
+            data = ''
+            try:
+                data = self.opener.open(url).read()
+            except Exception, e:
+                data = ''
+                logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+            if data is None:
+                raise StoryDoesNotExist("Problem reading story URL " + url + "!")
         
-        soup = bs.BeautifulStoneSoup(data)
+            if self.reqLoginData(data):
+                raise FailedToDownload("Error downloading Story: %s!  Login Failed!" % url)    
+        
+        soup = None
+        try:
+            soup = bs.BeautifulStoneSoup(data)
+        except:
+            raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % url)    
 
         self.storyName = ''
         self.authorName = ''
@@ -302,16 +322,29 @@ class PotionsNSnitches(FanfictionSiteAdapter):
     
         logging.debug('Getting data from: %s' % url)
     
-        data = self.opener.open(url).read()
+        data = ''
+        try:
+            data = self.opener.open(url).read()
+        except Exception, e:
+            data = ''
+            logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+        if data is None:
+            raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
         
         # need to do this, because for some reason the <br /> tag in the story causes problems
         data = data.replace('<br />', ' SOMETHING_BR ')
-        soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES)
-    
+
+        soup = None
+        try:
+            soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES)
+        except:
+            logging.info("Failed to decode: <%s>" % data)
+            raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
+        
         div = soup.find('div', {'id' : 'story'})
     
         if None == div:
-          return '<html/>'
+            raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         # put the <br /> tags back in..
         text = div.__str__('utf8').replace(' SOMETHING_BR ','<br />')    

twilighted.py

@@ -110,15 +110,36 @@ class Twilighted(FanfictionSiteAdapter):
 
     def extractIndividualUrls(self):
         url = self.url + '&chapter=1'
-        data = self.opener.open(url).read()
+
+        data = ''
+        try:
+            data = self.opener.open(url).read()
+        except Exception, e:
+            data = ''
+            logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+        if data is None:
+            raise StoryDoesNotExist("Problem reading story URL " + url + "!")
         
         if self.reqLoginData(data):
-          self.performLogin()
-          data = self.opener.open(url).read()
-          if self.reqLoginData(data):
-            return None
+            self.performLogin()
+
+            data = ''
+            try:
+                data = self.opener.open(url).read()
+            except Exception, e:
+                data = ''
+                logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+            if data is None:
+                raise StoryDoesNotExist("Problem reading story URL " + url + "!")
         
-        soup = bs.BeautifulStoneSoup(data)
+            if self.reqLoginData(data):
+                raise FailedToDownload("Error downloading Story: %s!  Login Failed!" % url)    
+        
+        soup = None
+        try:
+            soup = bs.BeautifulStoneSoup(data)
+        except:
+            raise FailedToDownload("Error downloading Story: %s!  Problem decoding page!" % url)    
 
         title = soup.find('title').string
         logging.debug('Title: %s' % title)
@@ -254,14 +275,26 @@ class Twilighted(FanfictionSiteAdapter):
     
         logging.debug('Getting data from: %s' % url)
     
-        data = self.opener.open(url).read()
+        data = ''
+        try:
+            data = self.opener.open(url).read()
+        except Exception, e:
+            data = ''
+            logging.error("Caught an exception reading URL " + url + ".  Exception " + str(e) + ".")
+        if data is None:
+            raise FailedToDownload("Error downloading Chapter: %s!  Problem getting page!" % url)
+                
+        soup = None
+        try:
+            soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES)
+        except:
+            logging.info("Failed to decode: <%s>" % data)
+            raise FailedToDownload("Error downloading Chapter: %s!  Problem decoding page!" % url)
         
-        soup = bs.BeautifulStoneSoup(data, convertEntities=bs.BeautifulStoneSoup.HTML_ENTITIES)
-    
         div = soup.find('div', {'id' : 'story'})
     
         if None == div:
-          return '<html/>'
+            raise FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
         return div.__str__('utf8')
 

zipdir.py

@@ -1,11 +1,79 @@
 # -*- coding: utf-8 -*-
 
+import sys
 import os
+import zlib
 import zipfile
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
+from contextlib import closing
 import logging
 
+import BeautifulSoup as bs
+import htmlentitydefs as hdefs
+import time
+import datetime
+from datetime import timedelta
+
 import StringIO
 
+class InvalidEPub(Exception):
+    pass
+
+def checkNewer(filename, curdte):
+	ret = True
+	
+	if not os.path.isfile(filename):
+		logging.debug('File %s does not already exist.' % filename)
+		return ret
+	
+	#logging.debug('filename=%s, curdte=%s' % (filename, curdte))
+	lastdate = None
+	with closing(ZipFile(open(filename, 'rb'))) as epub:
+		titleFilePath = "OEBPS/title_page.xhtml"
+		contentFilePath = "OEBPS/content.opf"
+		
+		namelist = set(epub.namelist())
+		#logging.debug('namelist=%s' % namelist)
+		if 'mimetype' not in namelist or \
+		   'META-INF/container.xml' not in namelist:
+			#raise InvalidEPub('%s: not a valid EPUB' % filename)
+			logging.debug('File %s is not a valid EPub format file.' % filename)
+			return ret
+		
+		if contentFilePath not in namelist:
+			return ret	# file is not newer
+		
+		data = epub.read(contentFilePath)
+		soup = bs.BeautifulStoneSoup(data)
+		lstdte = soup.find ('dc:date', {'opf:event' : 'modification'})
+		#logging.debug('lstdte=%s' % lstdte.string)
+		if lstdte is None and titleFilePath in namelist:
+			data = epub.read(titleFilePath)
+			soup = bs.BeautifulStoneSoup(data)
+			fld = ''
+			allTDs = soup.findAll ('td')
+			for td in allTDs:
+				b = td.find ('b')
+				if b is not None:
+					fld = b.string
+				if td.string is not None and fld == "Updated:":
+					lastdate = td.string
+					#logging.debug('title lastdate=%s' % lastdate)
+		else:
+			lastdate = lstdte.string.strip(' ')
+			#logging.debug('contents lastdate=%s' % lastdate)
+	
+	if lastdate is not None:	
+		currUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(curdte.strftime('%Y-%m-%d'), "%Y-%m-%d")))
+		storyUpdated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(lastdate, "%Y-%m-%d")))
+		logging.debug('File %s last update date is %s, comparing to %s' % (filename, storyUpdated, currUpdated))
+		if currUpdated <= storyUpdated :	
+			ret = False
+	
+	logging.debug("Does %s need to be updated? %s" % (filename, ret))
+	return ret
+
+
 def toZip(filename, directory):
 	zippedHelp = zipfile.ZipFile(filename, "w", compression=zipfile.ZIP_DEFLATED)
 	lst = os.listdir(directory)