Bump versions

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 14)
+    version             = (2, 2, 15)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/dialogs.py

@@ -108,6 +108,7 @@ save_collisions={
     OVERWRITE:SAVE_OVERWRITE,
     OVERWRITEALWAYS:SAVE_OVERWRITEALWAYS,
     CALIBREONLY:SAVE_CALIBREONLY,
+    CALIBREONLYSAVECOL:SAVE_CALIBREONLYSAVECOL,
     SAVE_SKIP:SKIP,
     SAVE_ADDNEW:ADDNEW,
     SAVE_UPDATE:UPDATE,
@@ -1456,6 +1457,8 @@ class ViewLog(SizePersistedDialog):
         ## error = (lineno, msg)
         for (lineno, error_msg) in errors:
             # print('adding label for error:%s: %s'%(lineno, error_msg))
+            if len(error_msg) > 200:
+                error_msg=error_msg[:200]+" ..."
             label = QLabel('%s: %s'%(lineno, error_msg))
             label.setWordWrap(True)
             label.setStyleSheet("QLabel { margin-left: 2em; color : blue; } QLabel:hover { color: red; }");

calibre-plugin/plugin-defaults.ini

@@ -344,12 +344,21 @@ anthology_title_pattern:${title} Anthology
 ## any anthology tags.
 anthology_tags:Anthology
 
-## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
-## on '/', so use replace_metadata to change separator first if
-## needed. Something like: ships=>[ ]*(/|&|&)[ ]*=>/ You can use
-## ships_LIST to change the / back to something else if you want.
+## Reorder ships so b/a and c/b/a become a/b and a/b/c. '/' is no
+## longer hard coded and can be changed and added to with
+## sort_ships_splits.
 sort_ships:false
 
+## Each line indicates first a regex that should be used to split each
+## ships entry and then, after => the string to use to merge the parts
+## back together.  \s == blank space.
+## Each part will have replace_metadata with key ships_CHARS applied.
+## By default, sort_ships_splits:/=>/
+## Here's possible an example for AO3:
+#sort_ships_splits:
+# [ ]*/[ ]*=>/
+# [ ]*&[ ]*=>\s&\s
+
 ## join_string_<entry> options -- FanFicFare list entries are comma
 ## separated by default. You can use this to change that. For example,
 ## if you want authors separated with ' & ' instead, use
@@ -709,7 +718,7 @@ extratags: FanFiction,Testing,HTML
 
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url,series00HTML,series01HTML,series02HTML,series03HTML
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -719,6 +728,14 @@ kudos_label:Kudos
 hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
+series00HTML_label:Series
+series01HTML_label:Additional Series
+series02HTML_label:Additional Series
+series03HTML_label:Additional Series
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:series00,series01,series02,series03
 
 ## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
 ## hardcoded to include the site specific metadata freeformtags &
@@ -737,7 +754,7 @@ include_in_category:fandoms
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series01HTML,series02HTML,series03HTML
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

calibre-plugin/translations/de.po

@@ -12,8 +12,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: calibre-plugins\n"
 "POT-Creation-Date: 2015-09-30 10:40+Central Daylight Time\n"
-"PO-Revision-Date: 2015-10-07 09:47+0000\n"
-"Last-Translator: Valsu <valsu@hylia.de>\n"
+"PO-Revision-Date: 2015-11-14 16:55+0000\n"
+"Last-Translator: ILB\n"
 "Language-Team: German (http://www.transifex.com/calibre/calibre-plugins/language/de/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -181,7 +181,7 @@ msgid ""
 "adult--stories that need those will just fail.<br />Only available for "
 "Update/Overwrite of existing books in case URL given isn't canonical or "
 "matches to existing book by Title/Author."
-msgstr ""
+msgstr "Bei jedem Download bietet FanFicFare eine Option, um im Hintergrund Metadaten von Webseiten zu sammeln.<br />Während eines Updates gibt dies die Kontrolle schneller an Sie zurück, aber Sie werden nicht nach Nutzernamen/Passwörtern oder ihrem Alter gefragt--Geschichten, die diese Daten benötigen, können nicht geladen werden.<br />Das Aktualisieren/Überschreiben vorhandener Bücher ist nur möglich, wenn die URL dem vorgegebenen Schema folgt oder mit dem Titel/Autor eines vorhandenen Buchs übereinstimmt."
 
 #: config.py:442
 msgid "Updating Calibre Options"
@@ -238,7 +238,7 @@ msgid ""
 "Check for existings Series Anthology books using each new story's series URL before downloading.\n"
 "Offer to skip downloading if a Series Anthology is found.\n"
 "Doesn't work when Collect Metadata in Background is selected."
-msgstr ""
+msgstr "Mit jeder neuen Serien-URL wird auf bereits vorhandene Serien-Sammelbände geprüft.\nFanFicFare bietet an, diese zu überspringen, wenn ein Sammelband gefunden wurde.\nFunktioniert nicht, wenn \"Aktualisiere Calibre Metadaten aus der gespeicherten Metadaten-Spalte\" selektiert ist."
 
 #: config.py:471
 msgid "Check for changed Story URL?"
@@ -270,7 +270,7 @@ msgid ""
 "Mark added/updated books when finished.  Use with option below.\n"
 "You can also manually search for 'marked:fff_success'.\n"
 "'marked:fff_failed' is also available, or search 'marked:fff' for both."
-msgstr ""
+msgstr "Markiere hinzugefügte/aktualisierte Bücher, wenn fertig. Verwendbar mit der nachfolgenden Option.\nSie können auch manuell nach 'marked:fff_success' (erfolgreich) suchen.\n'marked:fff_failed' (gescheitert) steht ebenfalls zur Verfügung, oder suche 'marked:fff' für beides."
 
 #: config.py:486
 msgid "Show Marked books when finished?"
@@ -281,7 +281,7 @@ msgid ""
 "Show Marked added/updated books only when finished.\n"
 "You can also manually search for 'marked:fff_success'.\n"
 "'marked:fff_failed' is also available, or search 'marked:fff' for both."
-msgstr ""
+msgstr "Zeige markierte hinzugefügte/aktualisierte Bücher, wenn fertig. \nSie können auch manuell nach 'marked:fff_success' (erfolgreich) suchen.\n'marked:fff_failed' (gescheitert) steht ebenfalls zur Verfügung, oder suche 'marked:fff' für beides."
 
 #: config.py:491
 msgid "Smarten Punctuation (EPUB only)"
@@ -599,7 +599,7 @@ msgstr ""
 
 #: config.py:857
 msgid "Generate Calibre Cover:"
-msgstr ""
+msgstr "Erzeuge Calibre Cover:"
 
 #: config.py:884
 msgid "Plugin %(gc)s"
@@ -1317,7 +1317,7 @@ msgid ""
 "control to you quicker while updating, but you won't be asked for "
 "username/passwords or if you are an adult--stories that need those will just"
 " fail."
-msgstr ""
+msgstr "Sammelt Metadaten von Webseiten im Hintergrund.\nWährend eines Updates gibt dies die Kontrolle schneller an Sie zurück, aber Sie werden nicht nach Nutzernamen/Passwörtern oder ihrem Alter gefragt--Geschichten, die diese Daten benötigen, können nicht geladen werden."
 
 #: dialogs.py:913 fff_plugin.py:1434 fff_plugin.py:1633 fff_plugin.py:1663
 msgid "Comment"
@@ -1894,7 +1894,7 @@ msgstr "Das existierende EPUB hat %d Kapitel, die Web-Seite nur %d. Benutzen Sie
 msgid ""
 "FanFicFare doesn't recognize chapters in existing epub, epub is probably "
 "from a different source. Use Overwrite to force update."
-msgstr ""
+msgstr "FanFicFare kann keine Kapitel im bestehenden epub erkennen, epub ist wahrscheinlich aus einer anderen Quelle. Verwenden Sie überschreiben, um die Aktualisierung zu erzwingen."
 
 #: fff_plugin.py:1312 jobs.py:208
 msgid "Not Overwriting, web site is not newer."
@@ -2036,25 +2036,25 @@ msgstr "Fehler"
 msgid ""
 "You configured FanFicFare to automatically update Reading Lists, but you "
 "don't have the %s plugin installed anymore?"
-msgstr ""
+msgstr "Sie konfigurierten FanFicFare Lese-Listen automatisch zu aktualisieren, aber Sie haben dieses % s Plugin nicht mehr installiert?"
 
 #: fff_plugin.py:2118
 msgid ""
 "You configured FanFicFare to automatically update \"To Read\" Reading Lists,"
 " but you don't have any lists set?"
-msgstr ""
+msgstr "Sie konfigurierten FanFicFare die \"zu lesen\"-Lese-Liste automatisch zu aktualisieren, aber Sie haben noch keine Listen festgelegt?"
 
 #: fff_plugin.py:2128 fff_plugin.py:2146
 msgid ""
 "You configured FanFicFare to automatically update Reading List '%s', but you"
 " don't have a list of that name?"
-msgstr ""
+msgstr "Sie konfigurierten FanFicFare Lese-Listen automatisch zu aktualisieren, aber Sie haben keine Listen mit diesem Namen?"
 
 #: fff_plugin.py:2134
 msgid ""
 "You configured FanFicFare to automatically update \"Send to Device\" Reading"
 " Lists, but you don't have any lists set?"
-msgstr ""
+msgstr "Sie konfigurierten FanFicFare die \"ans Gerät senden\"-Lese-Listen automatisch zu aktualisieren, aber Sie haben noch keine Listen festgelegt?"
 
 #: fff_plugin.py:2257
 msgid "No story URL found."

fanficfare/adapters/adapter_adastrafanficcom.py

@@ -137,7 +137,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ''
                 while value and not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 # sometimes poorly formated desc (<p> w/o </p>) leads
                 # to all labels being included.

fanficfare/adapters/adapter_ashwindersycophanthexcom.py

@@ -194,7 +194,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'table':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_bdsmgeschichten.py

@@ -265,7 +265,7 @@ class BdsmGeschichtenAdapter(BaseSiteAdapter):
                     nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
                     if nextLinkMatch is not None:
                         curChap = nextLinkMatch.group(1)
-                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
+                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, unicode(int(curChap) + 1), nextLink)
                     else:
                         break
             try:
@@ -289,7 +289,7 @@ class BdsmGeschichtenAdapter(BaseSiteAdapter):
         curMax = self.maxChapter + step
         lastHit = True
         while True:
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(curMax), self.url) 
             if nextChapterUrl == self.url:
                 logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
                 break;
@@ -316,8 +316,8 @@ class BdsmGeschichtenAdapter(BaseSiteAdapter):
             logger.debug(curMax)
 
         for i in xrange(1, self.maxChapter):
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
-            nextChapterTitle = re.sub("1", str(i), title1)
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(i), self.url)
+            nextChapterTitle = re.sub("1", unicode(i), title1)
             self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
 
     def getChapterText(self, url):

fanficfare/adapters/adapter_bloodtiesfancom.py

@@ -244,7 +244,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
                 ## Everything until the next strong tag.
                 svalue = ""
                 while not isinstance(value,bs.Tag) or value.name != 'strong':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_chaossycophanthexcom.py

@@ -156,7 +156,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             val = value
             value = value.previousSibling
         while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
 		

fanficfare/adapters/adapter_destinysgatewaycom.py

@@ -167,7 +167,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_dracoandginnycom.py

@@ -217,7 +217,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_dramioneorg.py

@@ -230,7 +230,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
                 ## Everything until the next strong tag.
                 svalue = ""
                 while not isinstance(value,bs.Tag) or value.name != 'strong':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_efictionestelielde.py

@@ -147,8 +147,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while 'Rating' not in str(value):
-                    svalue += str(value)
+                while 'Rating' not in unicode(value):
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -172,7 +172,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
             val = value
             value = value.previousSibling
         while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
 		

fanficfare/adapters/adapter_fanficcastletvnet.py

@@ -228,7 +228,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_ficbooknet.py

@@ -165,7 +165,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         i=0
         for part in pr:
             i=i+len(stripHTML(part).split(' '))
-        self.story.setMetadata('numWords', str(i))
+        self.story.setMetadata('numWords', unicode(i))
 		
         i=0
         fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))

fanficfare/adapters/adapter_ficwadcom.py

@@ -159,7 +159,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         ## perhaps not the most efficient way to parse this, using
         ## regexps for each rather than something more complex, but
         ## IMO, it's more readable and amenable to change.
-        metastr = stripHTML(str(metap)).replace('\n',' ').replace('\t',' ').replace(u'\u00a0',' ')
+        metastr = stripHTML(unicode(metap)).replace('\n',' ').replace('\t',' ').replace(u'\u00a0',' ')
 
         m = re.match(r".*?Rating: (.+?) -.*?",metastr)
         if m:

fanficfare/adapters/adapter_finestoriescom.py

@@ -160,7 +160,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         skip=0
         i=0
         while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+unicode(skip)))
         
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:

fanficfare/adapters/adapter_harrypotterfanfictioncom.py

@@ -113,7 +113,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
             tdstr = tr.findAll('td')[2].string
             if tdstr and tdstr.isdigit():
                 words+=int(tdstr)
-        self.story.setMetadata('numWords',str(words))
+        self.story.setMetadata('numWords',unicode(words))
         
         # Find the chapters:
         tablelist = soup.find('table',{'class':'text'})
@@ -135,7 +135,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         #for center in soup.findAll('center'):
         table = soup.find('table',{'class':'storymaininfo'})
         if table:
-            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            metastr = stripHTML(unicode(table)).replace('\n',' ').replace('\t',' ')
             # Rating: 12+ Story Reviews: 3
             # Chapters: 3
             # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC

fanficfare/adapters/adapter_hlfictionnet.py

@@ -147,7 +147,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_iketernalnet.py

@@ -222,7 +222,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_imagineeficcom.py

@@ -207,7 +207,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_ksarchivecom.py

@@ -195,7 +195,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     # poor HTML(unclosed <p> for one) can cause run on
                     # over the next label.
                     if '<span class="label">' in svalue:

fanficfare/adapters/adapter_literotica.py

@@ -237,12 +237,12 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
         # find num pages
         pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
-        logger.debug("pages: "+str(pgs))
+        logger.debug("pages: "+unicode(pgs))
 
         # get all the pages
         for i in xrange(2, pgs+1):
             try:
-                logger.debug("fetching page "+str(i))
+                logger.debug("fetching page "+unicode(i))
                 time.sleep(0.5)
                 data2 = self._fetchUrl(url, {'page': i})
                 # brute force approach to replace the wrapping <p> tag.  If

fanficfare/adapters/adapter_lumossycophanthexcom.py

@@ -156,7 +156,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             val = value
             value = value.previousSibling
         while not defaultGetattr(val,'class') == 'label':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
 		

fanficfare/adapters/adapter_masseffect2in.py

@@ -218,7 +218,7 @@ class MassEffect2InAdapter(BaseSiteAdapter):
         self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
         self.story.setMetadata('datePublished', datePublished)
         self.story.setMetadata('dateUpdated', dateUpdated)
-        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numWords', unicode(wordCount))
         self.story.setMetadata('numChapters', chapterCount)
 
         # Site-specific metadata.

fanficfare/adapters/adapter_mediaminerorg.py

@@ -41,6 +41,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         urltitle='urltitle'
+        cattitle='cattitle'
         if m:
             if m.group('id1'):
                 self.story.setMetadata('storyId',m.group('id1'))
@@ -50,13 +51,21 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 urltitle=m.group('urltitle2')
             elif m.group('id3'):
                 self.story.setMetadata('storyId',m.group('id3'))
+            elif m.group('id4'):
+                self.story.setMetadata('storyId',m.group('id4'))
+                cattitle=m.group('cattitle4')
+                urltitle=m.group('urltitle4')
+            elif m.group('id5'):
+                self.story.setMetadata('storyId',m.group('id5'))
+                cattitle=m.group('cattitle5')
+                urltitle=m.group('urltitle5')
             else:
                 raise InvalidStoryURL(url,
                                       self.getSiteDomain(),
                                       self.getSiteExampleURLs())
             
             # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/fanfic/s/'+urltitle+'/'+self.story.getMetadata('storyId'))
+            self._setURL('http://' + self.getSiteDomain() + '/fanfic/s/'+cattitle+'/'+urltitle+'/'+self.story.getMetadata('storyId'))
         else:
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
@@ -72,7 +81,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfic/s/story-title/123456 http://"+cls.getSiteDomain()+"/fanfic/c/story-title/chapter-title/123456/987612"
+        return "http://"+cls.getSiteDomain()+"/fanfic/s/category-name/story-title/123456 http://"+cls.getSiteDomain()+"/fanfic/c/category-name/story-title/123456/987612"
 
     def getSiteURLPattern(self):
         ## old urls
@@ -82,8 +91,13 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         ## http://www.mediaminer.org/fanfic/c/ghosts-from-the-past/chapter-2/72/174
         ## http://www.mediaminer.org/fanfic/s/robtech-final-missions/61553
         ## http://www.mediaminer.org/fanfic/c/robtech-final-missions/robotech-final-missions-oneshot/61553/189830
+        ## even newer urls
+        ## http://www.mediaminer.org/fanfic/s/gundam-wing-fan-fiction/the-preventer-operatives/171000
+        ## http://www.mediaminer.org/fanfic/c/gundam-wing-fan-fiction/the-preventer-operatives/171000/608822
         return re.escape("http://"+self.getSiteDomain())+r"/fanfic/"+\
-            r"((s/(?P<urltitle1>[^/]+)/(?P<id1>\d+))|"+\
+            r"((s/(?P<cattitle4>[^/]+)/(?P<urltitle4>[^/]+)/(?P<id4>\d+))|"+\
+            r"((c/(?P<cattitle5>[^/]+)/(?P<urltitle5>[^/]+)/(?P<id5>\d+))/\d+)|"+\
+            r"(s/(?P<urltitle1>[^/]+)/(?P<id1>\d+))|"+\
             r"((c/(?P<urltitle2>[^/]+)/[^/]+/(?P<id2>\d+))/\d+)|"+\
             r"(view_st\.php/(?P<id3>\d+)))"
             
@@ -109,40 +123,29 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = self.make_soup(data)
 
-        # [ A - All Readers ], strip '[' ']'
+        ## title:
+        ## <h1 id="post-title">A, A' Fan Fiction &#10095; Mmmmm</h1>
+        titletext = stripHTML(soup.find("h1",{"id":"post-title"}))
+        titletext = titletext[titletext.index(u'❯')+2:]
+        # print("title:(%s)"%titletext)
+        self.story.setMetadata('title',titletext)
+        
+        # [ A - All Readers ], strip '[ ' ' ]'
         ## Above title because we remove the smtxt font to get title.
-        smtxt = soup.find("h3",{"id":"post-rating"})
+        smtxt = soup.find("div",{"id":"post-rating"})
         if not smtxt:
             logger.error("can't find rating")
             raise exceptions.StoryDoesNotExist(self.url)
         else:
-            rating = smtxt.string[1:-1]
+            rating = smtxt.string[2:-2]
             self.story.setMetadata('rating',rating)
 
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+"))
+        a = soup.find('a', href=re.compile(r"/user_info.php/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[-1])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
-        ## Title - Good grief.  Title varies by chaptered, 1chapter and 'type=one shot'--and even 'one-shot's can have titled chapter.
-        ## But, if colspan=2, there's no chapter title.
-        ## <td class="ffh">Atmosphere: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td colspan=2 class="ffh">Hearts of Ice <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td colspan=2 class="ffh">Suzaku no Princess <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td class="ffh">The Kraut, The Bartender, and The Drunkard: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
-        ## <td class="ffh">Betrayal and Justice: A Cold Heart</b> <font size="-1">( Chapter 1 )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        ## <td class="ffh">Question and Answer: Question and Answer</b> <font size="-1">( One-Shot )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        # title = soup.find('td',{'class':'ffh'})
-        # for font in title.findAll('font'):
-        #     font.extract() # removes 'font' tags from inside the td.        
-        # if title.has_attr('colspan'):
-        #     titlet = stripHTML(title)
-        # else:
-        #     ## No colspan, it's part chapter title--even if it's a one-shot.
-        #     titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
-        self.story.setMetadata('title',stripHTML(soup.find('h1',{'id':'post-title'})))
-
         # save date from first for later.
         firstdate=None
         
@@ -155,7 +158,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
         # category
         # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
-        for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/a/")):
+        for a in soup.findAll('a',href=re.compile(r"^/fanfic/a/")):
             self.story.addToList('category',a.string)
         
         # genre

fanficfare/adapters/adapter_merlinficdtwinscouk.py

@@ -206,7 +206,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_midnightwhispersca.py

@@ -190,7 +190,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_mugglenetcom.py

@@ -240,7 +240,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             #     ## Everything until the next span class='label'
             #     svalue = ""
             #     while not defaultGetattr(value,'class') == 'label':
-            #         svalue += str(value)
+            #         svalue += unicode(value)
             #         value = value.nextSibling
             #     self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_nfacommunitycom.py

@@ -194,7 +194,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_occlumencysycophanthexcom.py

@@ -196,7 +196,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'table':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_onedirectionfanfictioncom.py

@@ -194,7 +194,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while value and not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_pommedesangcom.py

@@ -218,7 +218,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_ponyfictionarchivenet.py

@@ -181,7 +181,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         value = section.nextSibling
         svalue = ""
         while not defaultGetattr(value,'class') == 'label':
-            svalue += str(value)
+            svalue += unicode(value)
             value = value.nextSibling
         self.setDescription(url,svalue)
         

fanficfare/adapters/adapter_portkeyorg.py

@@ -202,7 +202,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
                 svalue = ""
                 value = labelspan.nextSibling
                 while not defaultGetattr(value,'class') == 'dark-small-bold':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                     
                 for genre in svalue.split("/"):

fanficfare/adapters/adapter_potionsandsnitches.py

@@ -123,7 +123,7 @@ class PotionsAndSnitchesOrgSiteAdapter(BaseSiteAdapter):
                 ## Everything until the next div class='listbox'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'listbox':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_potterficscom.py

@@ -245,7 +245,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         chnum = 0
         for li in list:
             chnum += 1
-            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
+            chTitle = unicode(chnum) + '. ' + li.a.b.string.strip()
             chURL = makeAbsoluteURL(li.a['href'])
             chapters.append((chTitle,chURL))
             #Get reviews, add to total
@@ -263,7 +263,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         mb = list.parent
         #get the div before that, will either be the description, or the google ad bar
         mb = mb.findPreviousSibling('div')
-        if 'google_ad_client' in str(mb):
+        if 'google_ad_client' in unicode(mb):
             #couldn't find description, leaving it blank
             pass
         else:

fanficfare/adapters/adapter_potterheadsanonymouscom.py

@@ -217,7 +217,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_pretendercentrecom.py

@@ -171,7 +171,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_psychficcom.py

@@ -166,7 +166,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_qafficcom.py

@@ -179,7 +179,7 @@ class QafFicComAdapter(BaseSiteAdapter):
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'classification' and value != None:
                     if "Featured Stories" not in value:
-                        svalue += str(value)
+                        svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_samdeanarchivenu.py

@@ -151,7 +151,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_scarheadnet.py

@@ -251,10 +251,10 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         svalue = ""
         while i == 0:
             try:
-                b = str(a)
+                b = unicode(a)
                 svalue += b.split('Summary: ')[1]
             except:
-                svalue += str(a)
+                svalue += unicode(a)
             if a.nextSibling != None:
                 a = a.nextSibling
             else:

fanficfare/adapters/adapter_scarvesandcoffeenet.py

@@ -175,7 +175,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_sg1heliopoliscom.py

@@ -176,7 +176,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_sinfuldesireorg.py

@@ -166,7 +166,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_squidgeorgpeja.py

@@ -158,7 +158,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'classification':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -178,7 +178,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         page=0
         i=0
         while i == 0:
-            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
+            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+unicode(page)))
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:

fanficfare/adapters/adapter_tenhawkpresentscom.py

@@ -172,7 +172,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_thealphagatecom.py

@@ -135,7 +135,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_themasquenet.py

@@ -215,7 +215,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_thepetulantpoetesscom.py

@@ -161,7 +161,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         index = 1
         found = 0
         while found == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+unicode(index)))
         
             for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
                 a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))

fanficfare/adapters/adapter_tolkienfanfiction.py

@@ -193,7 +193,7 @@ class TolkienFanfictionAdapter(BaseSiteAdapter):
             self.story.addToList('characters', character)
         logger.debug("Characters: %s" % self.story.getMetadata('characters'))
 
-        logger.debug('Title as `str`: ' + str(title))
+        logger.debug('Title as `str`: ' + unicode(title))
         # For publication date we need to search
         try:
             queryString = urllib.urlencode((

fanficfare/adapters/adapter_trekiverseorg.py

@@ -217,7 +217,7 @@ class TrekiverseOrgAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ''
                 while value and not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 # sometimes poorly formated desc (<p> w/o </p>) leads
                 # to all labels being included.

fanficfare/adapters/adapter_twcslibrarynet.py

@@ -170,7 +170,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     # poor HTML(unclosed <p> for one) can cause run on
                     # over the next label.
                     if '<span class="label">' in svalue:

fanficfare/adapters/adapter_twilightednet.py

@@ -162,7 +162,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
 

fanficfare/adapters/adapter_walkingtheplankorg.py

@@ -150,7 +150,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))

fanficfare/adapters/adapter_whoficcom.py

@@ -124,7 +124,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         # link instead to find the appropriate metadata.
         a = soup.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
         metadata = a.findParent('td')
-        metadatachunks = self.utf8FromSoup(None,metadata).split('<br />')
+        metadatachunks = self.utf8FromSoup(None,metadata,allow_replace_br_with_p=False).split('<br />')
         # process metadata for this story.
         self.setDescription(url,metadatachunks[1])
         #self.story.setMetadata('description', metadatachunks[1])

fanficfare/adapters/adapter_wolverineandroguecom.py

@@ -131,8 +131,8 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
         while value != None:
             val = value
             value = value.previousSibling
-        while "Categories" not in str(val):
-            svalue += str(val)
+        while "Categories" not in unicode(val):
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         

fanficfare/adapters/base_adapter.py

@@ -557,7 +557,7 @@ class BaseSiteAdapter(Configurable):
     # This gives us a unicode object, not just a string containing bytes.
     # (I gave soup a unicode string, you'd think it could give it back...)
     # Now also does a bunch of other common processing for us.
-    def utf8FromSoup(self,url,soup,fetch=None):
+    def utf8FromSoup(self,url,soup,fetch=None,allow_replace_br_with_p=True):
         if not fetch:
             fetch=self._fetchUrlRaw
 
@@ -616,7 +616,7 @@ class BaseSiteAdapter(Configurable):
         # This is primarily for epub updates.
         retval = re.sub(r"</?(html|head|body)[^>]*>\r?\n?","",retval)
         
-        if self.getConfig("replace_br_with_p"):
+        if self.getConfig("replace_br_with_p") and allow_replace_br_with_p:
             # Apply heuristic processing to replace <br> paragraph
             # breaks with <p> tags.
             retval = replace_br_with_p(retval)

fanficfare/adapters/base_efiction_adapter.py

@@ -338,7 +338,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                     errorDiv = soup.find("div", "errortext")
                     self.triedLoggingIn = True
                 else:
-                    raise exceptions.FailedToLogin(self.url, str(errorDiv))
+                    raise exceptions.FailedToLogin(self.url, unicode(errorDiv))
             else:
                 warningLink = errorDiv.find("a")
                 if warningLink is not None and ( \
@@ -354,9 +354,9 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                         errorDiv = soup.find("div", "errortext")
                         self.triedAcceptWarnings = True
                     else:
-                        raise exception.FailedToDownload(self.url, str(errorDiv))
+                        raise exception.FailedToDownload(self.url, unicode(errorDiv))
                 else:
-                    raise exception.FailedToDownload(self.url, str(errorDiv))
+                    raise exception.FailedToDownload(self.url, unicode(errorDiv))
 
         # title and author
         pagetitleDiv = soup.find("div", {"id": "pagetitle"})
@@ -382,7 +382,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 if (type(nextEl) is bs.Tag):
                     valueStr += nextEl.prettify()
                 else:
-                    valueStr += str(nextEl)
+                    valueStr += unicode(nextEl)
                 nextEl = nextEl.nextSibling
             key = labelSpan.text.strip()
 

fanficfare/cli.py

@@ -159,7 +159,7 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
                             options,
                             passed_defaultsini,
                             passed_personalini)
-                print("pagecache:%s"%options.pagecache.keys())
+                #print("pagecache:%s"%options.pagecache.keys())
             except Exception, e:
                 print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
     else:

fanficfare/configurable.py

@@ -302,6 +302,7 @@ def get_valid_keywords():
                  'replace_metadata',
                  'slow_down_sleep_time',
                  'sort_ships',
+                 'sort_ships_splits',
                  'strip_chapter_numbers',
                  'strip_chapter_numeral',
                  'strip_text_links',

fanficfare/defaults.ini

@@ -343,12 +343,21 @@ chapter_title_new_pattern:(new) ${title}
 ## (new) is just text and can be changed.
 chapter_title_addnew_pattern:${index}. (new) ${title}
 
-## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
-## on '/', so use replace_metadata to change separator first if
-## needed. Something like: ships=>[ ]*(/|&|&)[ ]*=>/ You can use
-## ships_LIST to change the / back to something else if you want.
+## Reorder ships so b/a and c/b/a become a/b and a/b/c. '/' is no
+## longer hard coded and can be changed and added to with
+## sort_ships_splits.
 sort_ships:false
 
+## Each line indicates first a regex that should be used to split each
+## ships entry and then, after => the string to use to merge the parts
+## back together.  \s == blank space.
+## Each part will have replace_metadata with key ships_CHARS applied.
+## By default, sort_ships_splits:/=>/
+## Here's possible an example for AO3:
+#sort_ships_splits:
+# [ ]*/[ ]*=>/
+# [ ]*&[ ]*=>\s&\s
+
 ## join_string_<entry> options -- FanFicFare list entries are comma
 ## separated by default. You can use this to change that. For example,
 ## if you want authors separated with ' & ' instead, use
@@ -715,7 +724,7 @@ extratags: FanFiction,Testing,HTML
 
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url,series00HTML,series01HTML,series02HTML,series03HTML
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -725,6 +734,14 @@ kudos_label:Kudos
 hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
+series00HTML_label:Series
+series01HTML_label:Additional Series
+series02HTML_label:Additional Series
+series03HTML_label:Additional Series
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:series00,series01,series02,series03
 
 ## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
 ## hardcoded to include the site specific metadata freeformtags &
@@ -743,7 +760,7 @@ include_in_category:fandoms
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series01HTML,series02HTML,series03HTML
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

fanficfare/epubutils.py

@@ -57,7 +57,7 @@ def get_update_data(inputio,
                 if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
                     oldcoverhtmltype = item.getAttribute("media-type")
                     break
-            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"))
+            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"),"html5lib")
             src = None
             # first img or image tag.
             imgs = soup.findAll('img')
@@ -208,7 +208,7 @@ def get_story_url_from_html(inputio,_is_good_url=None):
         #print("---- item:%s"%item)
         if( item.getAttribute("media-type") == "application/xhtml+xml" ):
             filehref=relpath+item.getAttribute("href")
-            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"))
+            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"),"html5lib")
             for link in soup.findAll('a',href=re.compile(r'^http.*')):
                 ahref=link['href']
                 #print("href:(%s)"%ahref)
@@ -265,34 +265,44 @@ def reset_orig_chapters_epub(inputio,outfile):
                 chapterorigtitle = None
                 tag = soup.find('meta',{'name':'chapterorigtitle'})
                 if tag:
-                    chapterorigtitle = tag['content'].replace('&','&').replace('"','"')
+                    chapterorigtitle = tag['content']
 
                 # toctitle is separate for add_chapter_numbers:toconly users.
                 chaptertoctitle = None
                 tag = soup.find('meta',{'name':'chaptertoctitle'})
                 if tag:
-                    chaptertoctitle = tag['content'].replace('&','&').replace('"','"')
-                elif chapterorigtitle:
+                    chaptertoctitle = tag['content']
                     chaptertoctitle = chapterorigtitle
 
                 chaptertitle = None
                 tag = soup.find('meta',{'name':'chaptertitle'})
                 if tag:
-                    chaptertitle = tag['content'].replace('&','&').replace('"','"')
+                    chaptertitle = tag['content']
+                    chaptertitle_tag = tag
 
-                # logger.debug("chaptertitle:(%s) chapterorigtitle:(%s)"%(chaptertitle, chapterorigtitle))
+                #logger.debug("chaptertitle:(%s) chapterorigtitle:(%s)"%(chaptertitle, chapterorigtitle))
                 if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
                     origdata = data
-                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
-                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
-                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
-                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
-                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+                    # data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                    #                     u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    # data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    # data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+                    chaptertitle_tag['content'] = chapterorigtitle
+                    title_tag = soup.find('title')
+                    if title_tag and title_tag.string == chaptertitle:
+                        title_tag.string.replace_with(chapterorigtitle)
+
+                    h3_tag = soup.find('h3')
+                    if h3_tag and h3_tag.string == chaptertitle:
+                        h3_tag.string.replace_with(chapterorigtitle)
+
+                    data = unicode(soup)
 
                     entrychanged = ( origdata != data )
                     changed = changed or entrychanged
 
                     if entrychanged:
+                        logger.debug("\nentrychanged:%s\n"%zf)
                         _replace_tocncx(tocncxdom,zf,chaptertoctitle)
                         ## Also look for and update individual
                         ## book toc.ncx files for anthology in case
@@ -331,7 +341,6 @@ def reset_orig_chapters_epub(inputio,outfile):
 
     return changed
 
-
 def _replace_tocncx(tocncxdom,zf,chaptertoctitle):
     ## go after the TOC entry, too.
     # <navPoint id="file0005" playOrder="6">

fanficfare/geturls.py

@@ -25,7 +25,7 @@ import urlparse
 import logging
 logger = logging.getLogger(__name__)
 
-from BeautifulSoup import BeautifulSoup 
+from bs4 import BeautifulSoup 
 from gziphttp import GZipProcessor
 
 import adapters
@@ -82,7 +82,7 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
-    soup = BeautifulSoup(data)
+    soup = BeautifulSoup(data,"html5lib")
     if restrictsearch:
         soup = soup.find(*restrictsearch)
         #logger.debug("restrict search:%s"%soup)

fanficfare/html.py

@@ -69,7 +69,7 @@ class HtmlProcessor:
     '''Replace <pre> tags with HTML-ified text.'''
     pres = self._soup.findAll('pre')
     for pre in pres:
-      pre.replaceWith(self._FixPreContents(str(pre.contents[0])))
+      pre.replaceWith(self._FixPreContents(unicode(pre.contents[0])))
 
   def _FixPreContents(self, text):
     if self.unfill:

fanficfare/story.py

@@ -77,7 +77,7 @@ try:
 
         if removetrans and img.has_transparent_pixels():
             canvas = Image()
-            canvas.create_canvas(int(img.size[0]), int(img.size[1]), str(background))
+            canvas.create_canvas(int(img.size[0]), int(img.size[1]), unicode(background))
             canvas.compose(img)
             img = canvas
             export = True
@@ -380,29 +380,34 @@ def set_in_ex_clude(setting):
 ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
 def make_replacements(replace):
     retval=[]
-    for line in replace.splitlines():
-        # print("replacement line:%s"%line)
-        (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
-        if "&&" in line:
-            (line,conditional) = line.split("&&")
-            (condkey,condregexp) = conditional.split("=>")
-        if "=>" in line:
-            parts = line.split("=>")
-            if len(parts) > 2:
-                metakeys = map( lambda x: x.strip(), parts[0].split(",") )
-                (regexp,replacement)=parts[1:]
-            else:
-                (regexp,replacement)=parts
-
-        if regexp:
-            regexp = re_compile(regexp,line)
-            if condregexp:
-                condregexp = re_compile(condregexp,line)
-            # A way to explicitly include spaces in the
-            # replacement string.  The .ini parser eats any
-            # trailing spaces.
-            replacement=replacement.replace(SPACE_REPLACE,' ')
-            retval.append([metakeys,regexp,replacement,condkey,condregexp])
+    for fullline in replace.splitlines():
+        line=fullline
+        try:
+            (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
+            if "&&" in line:
+                (line,conditional) = line.split("&&")
+                (condkey,condregexp) = conditional.split("=>")
+            if "=>" in line:
+                parts = line.split("=>")
+                if len(parts) > 2:
+                    metakeys = map( lambda x: x.strip(), parts[0].split(",") )
+                    (regexp,replacement)=parts[1:]
+                else:
+                    (regexp,replacement)=parts
+    
+            if regexp:
+                regexp = re_compile(regexp,line)
+                if condregexp:
+                    condregexp = re_compile(condregexp,line)
+                # A way to explicitly include spaces in the
+                # replacement string.  The .ini parser eats any
+                # trailing spaces.
+                replacement=replacement.replace(SPACE_REPLACE,' ')
+                retval.append([metakeys,regexp,replacement,condkey,condregexp])
+        except Exception as e:
+            logger.error("Problem with Replacement Line:%s"%fullline)
+            raise exceptions.PersonalIniFailed(e,'replace_metadata unpacking failed',fullline)
+#            raise
     return retval
 
 class Story(Configurable):
@@ -793,6 +798,33 @@ class Story(Configurable):
                 if retlist is None:
                     retlist = []
 
+        # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
+        # use replace_metadata to change separator first if needed.
+        # ships=>[ ]*(/|&|&)[ ]*=>/
+        if listname == 'ships' and self.getConfig('sort_ships') and doreplacements and retlist:
+            # retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
+            ## empty default of /=>/
+            sort_ships_splits = self.getConfig('sort_ships_splits',"/=>/")
+
+            for line in sort_ships_splits.splitlines():
+                if line:
+                    ## logger.debug("sort_ships_splits:%s"%line)
+                    ## logger.debug(retlist)
+                    (splitre,splitmerge) = line.split("=>")
+                    splitmerge = splitmerge.replace(SPACE_REPLACE,' ')
+                    newretlist = []
+                    for x in retlist:
+                        curlist = []
+                        for y in re.split(splitre,x):
+                            ## for SPLIT_META(\,)
+                            if doreplacements:
+                                y = self.doReplacements(y,'ships_CHARS',return_list=True)
+                            curlist.extend(y)
+                        newretlist.append( splitmerge.join(sorted(curlist)) )
+        
+                    retlist = newretlist
+                    ## logger.debug(retlist)
+        
         if retlist:
             if doreplacements:
                 newretlist = []
@@ -812,12 +844,6 @@ class Story(Configurable):
                                                                                                         )) > 1:
             retlist.append(self.getConfig('add_genre_when_multi_category'))
 
-        # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
-        # use replace_metadata to change separator first if needed.
-        # ships=>[ ]*(/|&|&)[ ]*=>/
-        if listname == 'ships' and self.getConfig('sort_ships') and retlist:
-            retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
-
         if retlist:
             if listname in ('author','authorUrl','authorId') or self.getConfig('keep_in_order_'+listname):
                 # need to retain order for author & authorUrl so the
@@ -1070,7 +1096,7 @@ class Story(Configurable):
         return retlist
 
     def __str__(self):
-        return "Metadata: " +str(self.metadata)
+        return "Metadata: " +unicode(self.metadata)
 
 def commaGroups(s):
     groups = []

fanficfare/translit.py

@@ -1,4 +1,4 @@
-#-*-coding:utf-8-*-
+#-*-coding:utf-8-*-
 # Code taken from http://python.su/forum/viewtopic.php?pid=66946
 import unicodedata
 def is_syllable(letter):
@@ -10,11 +10,11 @@ def is_consonant(letter):
     return not is_syllable(letter)
 def romanize(letter):
     try:
-        str(letter)
+        unicode(letter)
     except UnicodeEncodeError:
         pass
     else:
-        return str(letter)
+        return unicode(letter)
     unid = unicodedata.name(letter)
     exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
     for name_contains in exceptions:
@@ -54,4 +54,4 @@ def translit(text):
     #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
     #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
 if __name__=="__main__":
-    main()
+    main()

fanficfare/writers/writer_epub.py

@@ -578,7 +578,7 @@ div { margin: 0pt; padding: 0pt; }
             if title :
                 navPoint = newTag(tocncxdom,"navPoint",
                                   attrs={'id':id,
-                                         'playOrder':str(index)})
+                                         'playOrder':unicode(index)})
                 tocnavMap.appendChild(navPoint)
                 navLabel = newTag(tocncxdom,"navLabel")
                 navPoint.appendChild(navLabel)
@@ -653,7 +653,7 @@ div { margin: 0pt; padding: 0pt; }
         
         for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
             if chap.html:
-                logger.debug('Writing chapter text for: %s' % chap.title)
+                #logger.debug('Writing chapter text for: %s' % chap.title)
                 vals={'url':removeEntities(chap.url),
                       'chapter':removeEntities(chap.title),
                       'origchapter':removeEntities(chap.origtitle),

setup.py

@@ -23,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.14",
+    version="2.2.15",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-14
+version: 2-2-15
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/index.html

@@ -27,26 +27,6 @@
 	      much easier. </p>
 	  </div>
 
-	  <h3>Changes:</h3>
-	  <ul>
-            <li>Remove site thequidditchpitch.org, domain is parked.</li>
-            <li>Remove site devianthearts.com, server not found for a week.</li>
-            <li>Change site www.nickandgreg.net to www.nickngreg.nl while still accepting old URLs.</li>
-            <li>Change site ninelives.dark-solace.org to new domain ninelivesarchive.com.</li>
-            <li>Change site questionablequesting.com to forum.questionablequesting.com only.  Supporting both caused problems.</li>
-            <li>Fixes for hpfandom.net changes.</li>
-            <li>Fix for bloodshedverse.com.</li>
-            <li>Fixes for quotev.com (thanks, cryzed)</li>
-            <li>Fixes for base_xenforoforum adapters.</li>
-            <li>Add continue_on_chapter_error feature for base_xenforoforum adapters.</li>
-            <li>Add user/pass login for NSFW forums on forum.questionablequesting.com.</li>
-            <li>Add capitalize_forumtags feature to base_xenforoforum and include forumtags by default.</li>
-            <li>Account for base href in base_xenforoforum so emoticon images work.</li>
-            <li>Correct cover_exclusion_regexp for base_xenforoforum.</li>
-            <li>Change default HTML output to HTML5 header.</li>
-            <li>Update html tag header at beginning of each epub file.  Prompted by failure of old header on latest tolino ereader.</li>
-            <li>Fix replace_hr feature.</li>
-	  </ul>
 	  <p>
 	    Questions?  Check out our 
 	    <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
@@ -55,7 +35,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-13.fanficfare.appspot.com">previous version
+	    <a href="http://2-2-14.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>

webservice/main.py

@@ -153,7 +153,7 @@ class EditConfigServer(UserConfigServer):
                 self.redirect("/?error=configsaved")
             except Exception, e:
                 logging.info("Saved Config Failed:%s"%e)
-                self.redirect("/?error=custom&errtext=%s"%urlEscape(str(e)))
+                self.redirect("/?error=custom&errtext=%s"%urlEscape(unicode(e)))
         else: # not update, assume display for edit
             if uconfig is not None and uconfig.config:
                 config = uconfig.config
@@ -309,7 +309,7 @@ class RecentFilesServer(webapp2.RequestHandler):
 
         for fic in fics:
             if fic.completed and fic.format == 'epub':
-                fic.escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+str(fic.key())+"&fake=file."+fic.format)
+                fic.escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+unicode(fic.key())+"&fake=file."+fic.format)
 
         template_values = dict(fics = fics, nickname = user.nickname())
         path = os.path.join(os.path.dirname(__file__), 'recent.html')
@@ -391,7 +391,7 @@ class FanfictionDownloader(UserConfigServer):
             try:
                 configuration = self.getUserConfig(user,url,format)
             except Exception, e:
-                self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+str(e)))
+                self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+unicode(e)))
                 return
 
             adapter = adapters.getAdapter(configuration,url)
@@ -425,7 +425,7 @@ class FanfictionDownloader(UserConfigServer):
 
             taskqueue.add(url='/fdowntask',
                       queue_name="download",
-                          params={'id':str(download.key()),
+                          params={'id':unicode(download.key()),
                                   'format':format,
                                   'url':download.url,
                                   'login':login,
@@ -433,7 +433,7 @@ class FanfictionDownloader(UserConfigServer):
                                   'user':user.email(),
                                   'is_adult':is_adult})
 
-            logging.info("enqueued download key: " + str(download.key()))
+            logging.info("enqueued download key: " + unicode(download.key()))
 
         except (exceptions.FailedToLogin,exceptions.AdultCheckRequired), e:
             download.failure = unicode(e)
@@ -467,7 +467,7 @@ class FanfictionDownloader(UserConfigServer):
             download.failure = unicode(e)
             download.put()
 
-        self.redirect('/status?id='+str(download.key()))
+        self.redirect('/status?id='+unicode(download.key()))
 
         return