Plugin 1.1.5.

Existing Completed, In-Progress and Last Update tags still removed.
Add plugin-defaults.ini to remove unusable options and default lastupdate off.

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 1, 1)
+    version             = (1, 1, 5)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -27,6 +27,7 @@ prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 # Set defaults
 prefs.defaults['personal.ini'] = get_resources('example.ini')
 prefs.defaults['updatemeta'] = True
+prefs.defaults['keeptags'] = False
 #prefs.defaults['onlyoverwriteifnewer'] = False
 prefs.defaults['urlsfromclip'] = True
 prefs.defaults['updatedefault'] = True
@@ -72,10 +73,15 @@ class ConfigWidget(QWidget):
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setToolTip('Update title, author, URL, tags, etc for story in Calibre from web site.')
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
+        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
+        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setChecked(prefs['keeptags'])
+        self.l.addWidget(self.keeptags)
+
         # self.onlyoverwriteifnewer = QCheckBox('Default Only Overwrite Story if Newer',self)
         # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer or from the same day.")
         # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer'])
@@ -130,6 +136,7 @@ class ConfigWidget(QWidget):
         prefs['fileform'] = unicode(self.fileform.currentText())
         prefs['collision'] = unicode(self.collision.currentText())
         prefs['updatemeta'] = self.updatemeta.isChecked()
+        prefs['keeptags'] = self.keeptags.isChecked()
         prefs['urlsfromclip'] = self.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.updatedefault.isChecked()
 #        prefs['onlyoverwriteifnewer'] = self.onlyoverwriteifnewer.isChecked()
@@ -144,7 +151,7 @@ class ConfigWidget(QWidget):
             del prefs['personal.ini']
         
     def show_defaults(self):
-        text = get_resources('defaults.ini')
+        text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
 
     def reset_dialogs(self):

calibre-plugin/ffdl_plugin.py

@@ -286,7 +286,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         ## was self.ffdlconfig, but we need to be able to change it
         ## when doing epub update.
         ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
         ffdlconfig.readfp(StringIO(prefs['personal.ini']))
         adapter = adapters.getAdapter(ffdlconfig,url)
 
@@ -552,6 +552,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
                 if options['collision'] == CALIBREONLY or \
                         (options['updatemeta'] and book['good']) :
+                    if prefs['keeptags']:
+                        old_tags = db.get_tags(book['calibre_id'])
+                        # remove old Completed/In-Progress only if there's a new one.
+                        if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
+                            old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
+                        # remove old Last Update tags if there are new ones.
+                        if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
+                            old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+                        # mi.tags needs to be list, but set kills dups.
+                        mi.tags = list(set(list(old_tags)+mi.tags)) 
+                        
                     db.set_metadata(book['calibre_id'],mi)
                     
             add_list = filter(lambda x : x['good'] and x['added'], book_list)

calibre-plugin/jobs.py

@@ -103,14 +103,10 @@ def do_download_for_worker(book,options):
     when run as a worker job
     '''
     try:
-        # print("is_adult:%s"%book['is_adult'])
-        # print("personal.ini len:%s"%len(options['personal.ini']))
-        # print("defaults.ini len:%s"%len(get_resources("defaults.ini")))
-        #time.sleep(2.0)
         book['comment'] = 'Download started...'
 
         ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
         ffdlconfig.readfp(StringIO(options['personal.ini']))
         
         adapter = adapters.getAdapter(ffdlconfig,book['url'])

epubmerge.py

@@ -224,10 +224,21 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
                             pass # Skip missing files.
                 
         for itemref in metadom.getElementsByTagName("itemref"):
+
             if not striptitletoc or not re.match(r'(title|toc)_page', itemref.getAttribute("idref")):
                 itemrefs.append(bookid+itemref.getAttribute("idref"))
 
         booknum=booknum+1;
+        if not forceunique:
+            # If not forceunique, it's an epub update.
+            # If there's a "calibre_bookmarks.txt", it's from reading
+            # in Calibre and should be preserved.
+            try:
+                fn = "META-INF/calibre_bookmarks.txt"
+                outputepub.writestr(fn,epub.read(fn))
+            except:
+                pass
+        
 
     ## create content.opf file. 
     uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
@@ -355,7 +366,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
     ## during TOC generation to save loops.
     outputepub.writestr("content.opf",contentdom.toxml('utf-8'))
     outputepub.writestr("toc.ncx",tocncxdom.toxml('utf-8'))
-    
+
     # declares all the files created by Windows.  otherwise, when
     # it runs in appengine, windows unzips the files as 000 perms.
     for zf in outputepub.filelist:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -175,6 +175,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         a = soup.find('a', href='http://www.fictionratings.com/')
         self.story.setMetadata('rating',a.string)
 
+        # used below to get correct characters.
+        metatext = a.findNext(text=re.compile(r' - Reviews:'))
+        if metatext == None: # indicates there's no Reviews, look for id: instead.
+            metatext = a.findNext(text=re.compile(r' - id:'))
+        #print("========= metatext:\n%s"%metatext)
+
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
         if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
@@ -183,18 +189,33 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata('status', 'In-Progress')
 
         # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="Chapter 1 of a Harry Potter  - Family/Friendship fanfiction. Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter's wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story..">
-        # <meta name="description" content="A Gundam Wing/AC and Gundam Seed  - Romance/Sci-Fi crossover fanfiction  with characters:  & Kira Y.. Story summary: One-Shoot dividido en dos partes. Kira va en camino a rescatar a Lacus, pero él no es el unico. Dos personajes de diferentes universos Gundams. SEED vs ZERO.">
-        # <meta name="description" content="Chapter 1 of a Alvin and the chipmunks and Alpha and Omega  crossover fanfiction  with characters: Alvin S. & Humphrey. You'll just have to read to find out... No Flames Plesae... and tell me what you want to see by PM'ing me....">
-        # genre is after first -, but before first 'fanfiction'.
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?)) (?:crossover )?fanfiction",
+        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
+        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
+        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
+        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
+        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?:[ ]+with characters (?P<char1>.*?\.?)(?: & (?P<char2>.*?\.?))?\. )?",
                      soup.find('meta',{'name':'description'})['content'])
         if m != None:
             genres=m.group('genres')
-            # Hurt/Comfort is one genre.
-            genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-            for g in genres.split('/'):
-                self.story.addToList('genre',g)
+            if genres != None:
+                # Hurt/Comfort is one genre.
+                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
+                for g in genres.split('/'):
+                    self.story.addToList('genre',g)
+
+            if m.group('char1') != None:
+                # At this point we've proven that there's character(s)
+                # We can't reliably parse characters out of meta name="description".
+                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
+                # But we can pull them from the reviewstext line, now that we know about existance of chars.
+                # reviewstext can take form of:
+                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
+                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
+                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
+                              metatext)
+                chars = mc.group("chars")
+                for c in chars.split(' & '):
+                    self.story.addToList('characters',c)
         
         return
 

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -87,8 +87,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
-        
-        title, author = [link.text for link in soup.find("h2").findAll("a")]
+
+        titleheader = soup.find("h2")
+        title = titleheader.find("a", href=re.compile(r'^/story/')).text
+        author = titleheader.find("a", href=re.compile(r'^/user/')).text
         self.story.setMetadata("title", title)
         self.story.setMetadata("author", author)
         self.story.setMetadata("authorId", author) # The author's name will be unique
@@ -126,7 +128,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         status = status_bar.text.split("|")[0].strip().replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata('status', status)
         self.story.setMetadata('rating', status_bar.span.text)
-        self.story.setMetadata('numWords', status_bar.div.b.text)
+        # This way is less elegant, perhaps, but more robust in face of format changes.
+        numWords = status_bar.find("div",{"class":"word_count"}).b.text
+        self.story.setMetadata('numWords', numWords)
         
         description_soup = soup.find("div", {"class":"description"})
         # Sometimes the description has an expanding element
@@ -158,4 +162,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return utf8FromSoup(soup)
-        
+        

fanficdownloader/writers/base_writer.py

@@ -249,11 +249,18 @@ class BaseStoryWriter(Configurable):
     def getTags(self):
         # set to avoid duplicates subject tags.
         subjectset = set()
+        
+        if self.story.getMetadataRaw('dateUpdated'):
+            # Last Update tags for Bill.
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
+        
         for entry in self.validEntries:
             if entry in self.getConfigList("include_subject_tags") and \
                     entry not in self.story.getLists() and \
                     self.story.getMetadata(entry):
                 subjectset.add(self.getMetadata(entry))
+                
         # listables all go into dc:subject tags, but only if they are configured.
         for (name,lst) in self.story.getLists().iteritems():
             if name in self.getConfigList("include_subject_tags"):
@@ -263,7 +270,7 @@ class BaseStoryWriter(Configurable):
         for tag in self.getConfigList("extratags"):
             subjectset.add(tag)
 
-        return subjectset
+        return list(subjectset)
             
     def writeStoryImpl(self, out):
         "Must be overriden by sub classes."

fanficdownloader/writers/writer_epub.py

@@ -249,9 +249,6 @@ h6 { text-align: center; }
             metadata.appendChild(newTag(contentdom,"meta",
                                         attrs={"name":"calibre:timestamp",
                                                "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
         
         if self.getMetadata('description'):
             metadata.appendChild(newTag(contentdom,"dc:description",text=

makeplugin.py

@@ -27,7 +27,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
-                  ['defaults.ini','example.ini','epubmerge.py','fanficdownloader'],
+                  ['plugin-defaults.ini','example.ini','epubmerge.py','fanficdownloader'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')

plugin-defaults.ini

@@ -0,0 +1,242 @@
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels
+
+## All available titlepage_entries and the label used for them:
+## <entryname>_label:<label>
+## Labels may be customized.
+title_label:Title
+storyUrl_label:Story URL
+description_label:Summary
+author_label:Author
+authorUrl_label:Author URL
+## epub, txt, html
+formatname_label:File Format
+## .epub, .txt, .html
+formatext_label:File Extension
+## Category and Genre have overlap, depending on the site.
+## Sometimes Harry Potter is a category and Fantasy a genre. (fanfiction.net)
+## Sometimes Fantasy is category *and* a genre (fictionpress.com)
+## Sometimes there are multiple categories and/or genres.
+category_label:Category
+genre_label:Genre
+characters_label:Characters
+## Completed/In-Progress
+status_label:Status
+## Dates story first published, last updated, and downloaded(last with time).
+datePublished_label:Published
+dateUpdated_label:Updated
+dateCreated_label:Packaged
+## Rating depends on the site.  Some use K,T,M,etc, and some PG,R,NC-17
+rating_label:Rating
+## Also depends on the site.  
+warnings_label:Warnings
+numChapters_label:Chapters
+numWords_label:Words
+## www.fanfiction.net, fictionalley.com, etc.
+site_label:Publisher
+## ffnet, fpcom, etc.
+siteabbrev_label:Site Abbrev
+## The site's unique story/author identifier.  Usually a number.
+storyId_label:Story ID
+authorId_label:Author ID
+## Primarily to put specific values in dc:subject tags for epub.  Will
+## show up in Calibre as tags.  Also carried into mobi when converted.
+extratags_label:Extra Tags
+## The version of fanficdownloader
+##
+version_label:FFD Version
+
+## items to include in the title page
+## Empty entries will *not* appear, even if in the list.
+## All current formats already include title and author.
+titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## include title page as first page.
+include_titlepage: true
+
+## include a TOC page before the story text
+include_tocpage: true
+
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+#website_encodings: auto, utf8, Windows-1252
+
+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, status
+
+## extra tags (comma separated) to include, primarily for epub.
+extratags: FanFiction
+
+## number of seconds to sleep between calls to the story site.  May by
+## useful if pulling large numbers of stories or if the site is slow.
+## Primarily for commandline.
+#slow_down_sleep_time:0.5
+
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
+## Each output format has a section that overrides [defaults]
+[html]
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
+[epub]
+
+## epub is already a zip file.
+zip_output: false
+
+## epub carries the TOC in metadata.
+## mobi generated from epub will have a TOC at the end.
+include_tocpage: false
+
+## epub->mobi conversions typically don't like tables.
+titlepage_use_table: false
+
+## When using tables, make these span both columns.
+wide_titlepage_entries: description, storyUrl, author URL
+
+[mobi]
+## mobi TOC cannot be turned off right now.
+#include_tocpage: true
+
+
+## Each site has a section that overrides [defaults] *and* the format
+## sections test1.com specifically is not a real story site.  Instead,
+## it is a fake site for testing configuration and output.  It uses
+## URLs like: http://test1.com?sid=12345
+[test1.com]
+extratags: FanFiction,Testing
+
+## If necessary, you can define [<site>:<format>] sections to
+## customize the formats differently for the same site.  Overrides
+## defaults, format and site.
+[test1.com:txt]
+extratags: FanFiction,Testing,Text
+
+[test1.com:html]
+extratags: FanFiction,Testing,HTML
+
+[www.fanfiction.net]
+
+[www.fictionpress.com]
+## Clear FanFiction from defaults, fictionpress.com is original fiction.
+extratags:
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.whofic.com]
+
+[www.mediaminer.org]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fictionally.org storyIds are not unique.  Combine with authorId.
+output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[overrides]
+## It may sometimes be useful to override all of the specific format,
+## site and site:format sections in your private configuration.  For
+## example, this extratags param here would override all of the
+## extratags params in all other sections.  Only commandline options
+## beat overrides.
+#extratags:fanficdownloader