Change the app name back. (sigh)

& plugin.  Save plugin bump to 1.3.1.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-2-1
+version: 4-3-0
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 3, 0)
+    version             = (1, 3, 1)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -435,8 +435,10 @@ permitted_values = {
     'float' : ['numWords','numChapters'],
     'bool' : ['status-C','status-I'],
     'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
+    'series' : ['series'],
     'enumeration' : ['category',
                      'genre',
+                     'series',
                      'characters',
                      'status',
                      'datePublished',
@@ -471,6 +473,7 @@ titleLabels = {
     'status':'Status',
     'status-C':'Status:Completed',
     'status-I':'Status:In-Progress',
+    'series':'Series',
     'characters':'Characters',
     'datePublished':'Published',
     'dateUpdated':'Updated',
@@ -491,7 +494,7 @@ titleLabels = {
     'formatname':'File Format',
     'formatext':'File Extension',
     'siteabbrev':'Site Abbrev',
-    'version':'FFD Version'
+    'version':'FFDL Version'
     }
 
 class ColumnsTab(QWidget):

calibre-plugin/dialogs.py

@@ -152,7 +152,7 @@ class AddNewDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
     def get_urlstext(self):
@@ -417,7 +417,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         return {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
-            'updatemeta': unicode(self.updatemeta.isChecked()),
+            'updatemeta': self.updatemeta.isChecked(),
             }
 
 def display_story_list(gui, header, prefs, icon, books,

calibre-plugin/ffdl_plugin.py

@@ -414,6 +414,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['publisher'] = story.getMetadata("site")
         book['tags'] = writer.getTags()
         book['comments'] = story.getMetadata("description") #, removeallentities=True) comments handles entities better.
+        book['series'] = story.getMetadata("series")
         
         # adapter.opener is the element with a threadlock.  But del
         # adapter.opener doesn't work--subproc fails when it tries
@@ -425,7 +426,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
-        book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
+        if story.getMetadataRaw('datePublished'):
+            # should only happen when an adapter is broken, but better to
+            # fail gracefully.
+            book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
         book['timestamp'] = None # filled below if not skipped.
         
         if collision in (CALIBREONLY):
@@ -598,9 +602,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self._add_or_update_book(book,options,prefs,mi)
 
         if options['collision'] == CALIBREONLY or \
-                (options['updatemeta'] and book['good']) :
-            self._update_metadata(db, book['calibre_id'], book, mi)
-        
+                (options['updatemeta'] and book['good']):
+            self._update_metadata(db, book['calibre_id'], book, mi)        
 
     def _update_books_completed(self, book_list, options={}):
         
@@ -663,49 +666,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                init_label="Updating calibre for stories...",
                                win_title="Update calibre for stories",
                                status_prefix="Updated")
-            
-            # for book in good_list:
-            #     print("add/update %s %s"%(book['title'],book['url']))
-            #     mi = self._make_mi_from_book(book)
-                
-            #     if options['collision'] != CALIBREONLY:
-            #         self._add_or_update_book(book,options,prefs,mi)
-
-            #     if options['collision'] == CALIBREONLY or \
-            #             (options['updatemeta'] and book['good']) :
-            #         self._update_metadata(db, book['calibre_id'], book, mi)
-
-            ##### split here.
-            
-        #     add_list = filter(lambda x : x['good'] and x['added'], book_list)
-        #     update_list = filter(lambda x : x['good'] and not x['added'], book_list)
-        #     update_ids = [ x['calibre_id'] for x in update_list ]
-                    
-        #     if len(add_list):
-        #         ## even shows up added to searchs.  Nice.
-        #         self.gui.library_view.model().books_added(len(add_list))
-
-        #     if update_ids:
-        #         self.gui.library_view.model().refresh_ids(update_ids)
-
-        #     current = self.gui.library_view.currentIndex()
-        #     self.gui.library_view.model().current_changed(current, previous)
-        #     self.gui.tags_view.recount()
-            
-        #     self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
-        #     if len(update_list) + len(add_list) != total_good:
-        #         d = DisplayStoryListDialog(self.gui,
-        #                                    'Updates completed, final status',
-        #                                    prefs,
-        #                                    self.qaction.icon(),
-        #                                    book_list,
-        #                                    label_text='Stories have be added or updated in Calibre, some had additional problems.'
-        #                                    )
-        #         d.exec_()
-    
-        # print("all done, remove temp dir.")
-        # remove_dir(options['tdir'])
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -776,7 +736,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 print("%s not a valid column type for %s, skipping."%(col,meta))
                 continue
             label = coldef['label']
-            if coldef['datatype'] in ('enumeration','text','comments','datetime'):
+            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
                 db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
@@ -863,6 +823,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         mi.pubdate = book['pubdate']
         mi.timestamp = book['timestamp']
         mi.comments = book['comments']
+        mi.series = book['series']
         return mi
 
 
@@ -908,12 +869,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['title'] = mi.title
         book['author'] = authors_to_string(mi.authors)
         book['author_sort'] = mi.author_sort
-        # book['series'] = mi.series
-        # if mi.series:
-        #     book['series_index'] = mi.series_index
-        # else:
-        #     book['series_index'] = 0
-            
         book['comment'] = ''
         book['url'] = ""
         book['added'] = False

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2011 Fanficdownloader team
+# Copyright 2012 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ formatext_label:File Extension
 category_label:Category
 genre_label:Genre
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +62,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version
 
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -129,12 +137,13 @@ background_color: ffffff
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -197,6 +206,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twiwrite.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -205,6 +217,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.whofic.com]
 
 [www.mediaminer.org]
@@ -222,6 +237,9 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -262,6 +280,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
 [overrides]
 ## It may sometimes be useful to override all of the specific format,

editconfig.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main' style="width: 80%; margin-left: 10%;">
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -67,7 +67,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>
 

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -174,13 +174,33 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
             
             if 'Updated' in label:
                 # there's a stray [ at the end.
                 #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -272,6 +272,26 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -176,7 +176,27 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -81,7 +81,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+r"(www\.)?"+re.escape("siye.co.uk/siye/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
 
     # ## Login seems to be reasonably standard across eFiction sites.
     # def needToLoginCheck(self, data):
@@ -228,15 +228,13 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if part.startswith("Summary:"):
                 part = part[part.find(':')+1:]
                 self.story.setMetadata('description',part)
-
-        
-
+                
         # want to get the next tr of the table.
         #print("%s"%titlea.parent.parent.findNextSibling('tr'))
             
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
-        moremeta = stripHTML(titlea.parent.parent.findNextSibling('tr'))
+        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
         for part in moremeta.replace(' - ','\n').split('\n'):
             #print("part:%s"%part)
             try:
@@ -259,7 +257,25 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if name == 'Words':
                 self.story.setMetadata('numWords', value)
 
-        
+        try:
+            # Find Series name from series URL.
+            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass        
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -205,6 +205,26 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_test1.py

@@ -84,7 +84,15 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
-        self.story.setMetadata('status','In-Completed')
+
+        idnum = int(self.story.getMetadata('storyId'))
+        if idnum % 2 == 1:
+            self.story.setMetadata('status','In-Progress')
+        else:
+            self.story.setMetadata('status','Completed')
+            
+        self.setSeries('The Great Test',idnum)
+        
         self.story.setMetadata('rating','Tweenie')
         
         self.story.setMetadata('authorId','98765')

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -207,7 +207,26 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/library/'+a['href']
 
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -214,6 +214,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 self.chapterUrls.append((stripHTML(o),url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        pseries = soup.find('p', {'style':'margin-top:0px'})
+        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
+                     pseries.text)
+        if m:
+            self.setSeries(m.group('series'),m.group('num'))
         
         return
 

fanficdownloader/adapters/adapter_twilightednet.py

@@ -204,6 +204,25 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+            
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+        
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -217,6 +217,25 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))
 
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_whoficcom.py

@@ -189,6 +189,26 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             if name == 'Word Count':
                 self.story.setMetadata('numWords', value)
 
+        try:
+            # Find Series name from series URL.
+            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/base_adapter.py

@@ -229,6 +229,11 @@ class BaseSiteAdapter(Configurable):
         "Needs to be overriden in each adapter class."
         pass
 
+    # Just for series, in case we choose to change how it's stored or represented later.
+    def setSeries(self,name,num):
+        if self.getConfig('collect_series'):
+            self.story.setMetadata('series','%s [%s]'%(name, num))
+
 fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }

fanficdownloader/writers/base_writer.py

@@ -50,6 +50,7 @@ class BaseStoryWriter(Configurable):
             'category',
             'genre',
             'characters',
+            'series',
             'status',
             'datePublished',
             'dateUpdated',
@@ -77,6 +78,7 @@ class BaseStoryWriter(Configurable):
             'category':'Category',
             'genre':'Genre',
             'status':'Status',
+            'series':'Series',
             'characters':'Characters',
             'datePublished':'Published',
             'dateUpdated':'Updated',
@@ -97,7 +99,7 @@ class BaseStoryWriter(Configurable):
             'formatname':'File Format',
             'formatext':'File Extension',
             'siteabbrev':'Site Abbrev',
-            'version':'FFD Version'
+            'version':'FFDL Version'
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())

index-ajax.html

@@ -4,7 +4,7 @@
 		<link href="css/index.css" rel="stylesheet" type="text/css">
 		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
 	 
-		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<title>FanFictionDownLoader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 		<script src="/js/jquery-1.3.2.js"></script>
 		<script src="/js/fdownloader.js"></script>
@@ -16,7 +16,7 @@
 	<body>
 		<div id='main'>
 			<h1>
-				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+				<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
 			</h1>
 				
 <!--			<form action="/fdown" method="post"> -->
@@ -91,8 +91,8 @@
 					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 					alt="Powered by Google App Engine" />
 					<br/><br/>
-					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; Fanficdownloader team
 				</div>
 <!--			</form> -->
 		</div>

index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -26,7 +26,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a> <g:plusone size="medium"></g:plusone>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a> <g:plusone size="medium"></g:plusone>
       </h1>
 
       <div style="text-align: center">
@@ -50,7 +50,7 @@
       <form action="/fdown" method="post">
 	<div id='urlbox'>
 	  <div id='greeting'>
-	    <p>Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
@@ -61,47 +61,29 @@
 	    considers Python 2.7 Experimental still, so there may be issues.
 	  </p>
 	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <b>Support for 'Series'</b>
 	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
+	    We now collect 'Series' name and number for the sites:
+	      harrypotterfanfiction.com,
+	      potionsandsnitches.net,
+	      adastrafanfic.com,
+	      whofic.com,
+	      fanfiction.tenhawkpresents.com,
+	      castlefans.org,
+	      tthfanfic.org,
+	      www.siye.co.uk,
+	      twilighted.net*,
+	      twilighted.net* and 
+	      thewriterscoffeeshop.com*.
 	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-	  <p>
-	    <b>Wider support for 'Characters'</b>
-	    <br /><br />
-	    We now collect 'Character' lists for most of the supported sites.  
-	    <br /><br />
-	    There's huge variation in how different sites choose to define characters--well beyond our ability to standardize.
-	    <br /><br />
-	    So if you don't like and don't want to see characters in your title pages or tags, you can turn them off in your configuration with:
-	    <pre>
-[defaults]
-# note that characters is removed.  This removes it from your title page.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
-
-# note that characters is removed.  This removes it from your tags.
-include_subject_tags: extratags, genre, category, status
-	    </pre>
-	    If you like them for some sites, but not others, just copy the titlepage_entries and include_subject_tags to sections for different sites and edit to taste.
+	    * The last three use series as reading lists and stories collections as much as true story series,
+	    so they default to <i>not</i> collect series info.  You can turn it on in your User Configuration if you want.
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
-	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.  The
-	    <a href="http://4-2-0.fanfictiondownloader.appspot.com">Previous
-	      Version</a> is also available for you to use if necessary.
+	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
+	    <a href="http://4-2-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -137,14 +119,33 @@ include_subject_tags: extratags, genre, category, status
       <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
-	    can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+	    This is a FanFictionDownLoader, which makes reading stories from various websites much easier. Before you
+	    can start downloading fanfics, you need to login, so FanFictionDownLoader can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
 	</div>
       </div>
       {% endif %}
       
+      <div id='typebox'>
+	  <p>
+	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <br /><br />
+	    
+	    There's now a version of this downloader that runs
+	    entirely inside the
+	    popular <a href="http://calibre-ebook.com/">calibre</a>
+	    ebook management package as a plugin.
+	    
+	    <br /><br />
+	    
+	    Once you have calibre installed and running, inside
+	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    'Get new plugins' and
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
+	    
+	  </p>
+      </div>
       <div id='helpbox'>
 	<dl>
 	  <dt>fictionalley.org</dt>
@@ -281,8 +282,8 @@ include_subject_tags: extratags, genre, category, status
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

login.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Login Needed Fanfiction Downloader</title>
+    <title>Login Needed FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     <script type="text/javascript">
@@ -22,7 +22,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
 
       <div style="text-align: center">
@@ -88,8 +88,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
       </div>
 
       <div style="margin-top: 1em; text-align: center'">

plugin-defaults.ini

@@ -42,6 +42,7 @@ formatext_label:File Extension
 category_label:Category
 genre_label:Genre
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -71,7 +72,14 @@ version_label:FFDL Version
 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
 
 ## include title page as first page.
 include_titlepage: true
@@ -108,7 +116,7 @@ background_color: ffffff
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -168,6 +176,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.twiwrite.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -176,6 +187,9 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.whofic.com]
 
 [www.mediaminer.org]
@@ -193,6 +207,9 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

plugin-example.ini

@@ -7,11 +7,23 @@
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+## Turn off in [defaults] or [overrides] to prevent all sites from
+## updating series column.
+## default is true
+#collect_series: false
+
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
@@ -20,6 +32,8 @@
 [www.twiwrite.net]
 #username:YourName
 #password:yourpassword
+## default is false
+#collect_series: true
 
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
@@ -30,6 +44,8 @@
 #username:YourName
 #password:yourpassword
 #is_adult:true
+## default is false
+#collect_series: true
 
 [www.fictionalley.org]
 #is_adult:true
@@ -53,3 +69,6 @@
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

recent.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <script type="text/javascript">
 
@@ -21,7 +21,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       
       <script type="text/javascript"><!--
@@ -46,26 +46,6 @@
 	</div>
       </div>
 
-      <div id='typebox'>
-	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
-	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
-	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-      </div>
-      
       <div id='helpbox'>
 	{% for fic in fics %}
 	<p>

status.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - Fanfiction Downloader</title>
+    <title>{% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - FanFictionDownLoader</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
     {% if not fic.completed and not fic.failure %}
@@ -25,7 +25,7 @@
   <body>
     <div id='main'>
       <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
       </h1>
       <div style="text-align: center">
         <script type="text/javascript"><!--
@@ -68,30 +68,11 @@
 	  <p>See your personal list of <a href="/recent">previously downloaded fanfics</a>.</p>
       </div>
 
-      <div id='typebox'>
-	  <p>
-	    <b>FanFictionDownLoader calibre Plugin</b>
-	    <br /><br />
-	    
-	    There's now a version of this downloader that runs
-	    entirely inside the
-	    popular <a href="http://calibre-ebook.com/">calibre</a>
-	    ebook management package as a plugin.
-	    
-	    <br /><br />
-	    
-	    Once you have calibre installed and running, inside
-	    calibre, you can go to 'Get plugins to enhance calibre' or
-	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
-	    
-	  </p>
-      </div>
       <div style='text-align: center'>
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
       </div>