Fixes for: thehexfiles nasty HTML, spaces in img urls, status 'updating BAD'

Bump web service number.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-17
+version: 4-4-18
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 38)
+    version             = (1, 5, 41)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -61,6 +61,7 @@ all_prefs.defaults['gcnewonly'] = False
 all_prefs.defaults['gc_site_settings'] = {}
 all_prefs.defaults['allow_gc_from_ini'] = True
 
+all_prefs.defaults['errorcol'] = ''
 all_prefs.defaults['custom_cols'] = {}
 
 # The list of settings to copy from all_prefs or the previous library
@@ -164,7 +165,7 @@ class ConfigWidget(QWidget):
         if 'Generate Cover' not in plugin_action.gui.iactions:
             self.generatecover_tab.setEnabled(False)
 
-        self.columns_tab = ColumnsTab(self, plugin_action)
+        self.columns_tab = CustomColumnsTab(self, plugin_action)
         tab_widget.addTab(self.columns_tab, 'Custom Columns')
 
         self.other_tab = OtherTab(self, plugin_action)
@@ -218,6 +219,11 @@ class ConfigWidget(QWidget):
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
 
         # Custom Columns tab
+
+        # error column
+        prefs['errorcol'] = unicode(self.columns_tab.errorcol.itemData(self.columns_tab.errorcol.currentIndex()).toString())
+
+        # cust cols
         colsmap = {}
         for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -661,13 +667,15 @@ titleLabels = {
     'version':'FFDL Version'
     }
 
-class ColumnsTab(QWidget):
+class CustomColumnsTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
         self.parent_dialog = parent_dialog
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
@@ -678,8 +686,6 @@ class ColumnsTab(QWidget):
         
         self.custcol_dropdowns = {}
 
-        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
-
         for key, column in custom_columns.iteritems():
 
             if column['datatype'] in permitted_values:
@@ -707,4 +713,26 @@ class ColumnsTab(QWidget):
         
         self.l.insertStretch(-1)
 
+        self.l.addSpacing(5)
+        label = QLabel("Special column:")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        horz = QHBoxLayout()
+        label = QLabel("Update/Overwrite Error Column:")
+        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.errorcol = QComboBox(self)
+        self.errorcol.setToolTip(tooltip)
+        self.errorcol.addItem('',QVariant('none'))
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('text','comments'):
+                self.errorcol.addItem(column['name'],QVariant(key))
+        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+        horz.addWidget(self.errorcol)
+        self.l.addLayout(horz)
+        
+        
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/ffdl_plugin.py

@@ -653,6 +653,23 @@ make_firstimage_cover:true
                                        label_text='None of the URLs/stories given can be/need to be downloaded.'
                                        )
             d.exec_()
+
+            
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                label = custom_columns[prefs['errorcol']]['label']
+                print("errorcol label:%s"%label)
+                ## if error column and all bad.
+                self.previous = self.gui.library_view.currentIndex()
+                LoopProgressDialog(self.gui,
+                                   book_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
+                                   status_prefix="Updated")
+            
+            
             return
             
         func = 'arbitrary_n'
@@ -682,7 +699,16 @@ make_firstimage_cover:true
                 (options['updatemeta'] and book['good']):
             self._update_metadata(db, book['calibre_id'], book, mi, options)
 
-    def _update_books_completed(self, book_list, options={}):
+    def _update_bad_book(self,book,db=None,label='errorcol',
+                         options={'fileform':'epub',
+                                  'collision':ADDNEW,
+                                  'updatemeta':True,
+                                  'updateepubcover':True},):
+        
+        print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+        db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+
+    def _update_books_completed(self, book_list, options={}, showlist=True):
         
         add_list = filter(lambda x : x['good'] and x['added'], book_list)
         add_ids = [ x['calibre_id'] for x in add_list ]
@@ -706,7 +732,7 @@ make_firstimage_cover:true
         
         self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
             
-        if len(update_list) + len(add_list) != len(book_list):
+        if showlist and (len(update_list) + len(add_list) != len(book_list)):
             d = DisplayStoryListDialog(self.gui,
                                        'Updates completed, final status',
                                        prefs,
@@ -750,6 +776,25 @@ make_firstimage_cover:true
                                    win_title="Update calibre for FanFiction stories",
                                    status_prefix="Updated")
 
+            bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
+            total_bad = len(bad_list)
+
+            if total_bad > 0:
+                custom_columns = self.gui.library_view.model().custom_columns
+                if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                    self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
+                    label = custom_columns[prefs['errorcol']]['label']
+                    print("errorcol label:%s"%label)
+                    ## if error column and all bad.
+                    LoopProgressDialog(self.gui,
+                                       bad_list,
+                                       partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                       partial(self._update_books_completed, options=options, showlist=False),
+                                       init_label="Updating calibre for BAD FanFiction stories...",
+                                       win_title="Update calibre for BAD FanFiction stories",
+                                       status_prefix="Updated")
+
+                
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
         
@@ -888,7 +933,12 @@ make_firstimage_cover:true
                 print("Running Generate Cover with settings %s."%setting_name)
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
-        
+
+        ## if error column set.
+        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+            label = custom_columns[prefs['errorcol']]['label']
+            print("_update_book label:%s"%label)
+            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
 
     def _get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :

defaults.ini

@@ -66,7 +66,9 @@ extratags_label:Extra Tags
 version_label:FFDL Version
 
 ## items to include in the title page
-## Empty entries will *not* appear, even if in the list.
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
 titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
@@ -311,6 +313,19 @@ extratags: FanFiction,Testing,HTML
 
 [archive.skyehawke.com]
 
+[archiveofourown.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -398,6 +413,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[national-library.net]
+
+[ncisfic.com]
+
 [nfacommunity.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -464,12 +483,6 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [www.thealphagate.com]
 
-[www.archiveofourown.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.checkmated.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -492,6 +505,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 never_make_cover: true
 
 [www.fanfiktion.de]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
 [www.ficbook.net]
 

fanficdownloader/adapters/__init__.py

@@ -79,6 +79,8 @@ import adapter_thealphagatecom
 import adapter_fanfiktionde
 import adapter_ponyfictionarchivenet
 import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -42,6 +42,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                                # utf8) are really windows-1252.
 							   
 							   
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
@@ -60,13 +63,54 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.archiveofourown.org'
+        return 'archiveofourown.org'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.archiveofourown.org','archiveofourown.org']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        
+    ## Login
+    def needToLoginCheck(self, data):
+        if 'This work is only available to registered users of the Archive.' in data \
+                or "The password or user name you entered doesn't match our records" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url, data):
+
+        params = {}
+        if self.password:
+            params['user_session[login]'] = self.username
+            params['user_session[password]'] = self.password
+        else:
+            params['user_session[login]'] = self.getConfig("username")
+            params['user_session[password]'] = self.getConfig("password")
+        params['user_session[remember_me]'] = '1'
+        params['commit'] = 'Log in'
+        #params['utf8'] = u'✓'#u'\x2713' # gets along with out it, and it confuses the encoder.
+        params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user_session[login]']))
+    
+        d = self._postUrl(loginUrl, params)
+        #logging.info(d)
+    
+        if "Successfully logged in" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user_session[login]']))
+            raise exceptions.FailedToLogin(url,params['user_session[login]'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -76,13 +120,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         else:
             addurl=""
 
-        meta = self.url+addurl
+        metaurl = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logging.debug("URL: "+meta)
+        logging.info("url: "+url)
+        logging.info("metaurl: "+metaurl)
 
         try:
             data = self._fetchUrl(url)
-            meta = self._fetchUrl(meta)
+            meta = self._fetchUrl(metaurl)
 
             if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
                 raise exceptions.AdultCheckRequired(self.url)
@@ -92,11 +137,16 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.meta)
             else:
                 raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,data)
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(metaurl)
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         metasoup = bs.BeautifulSoup(meta)
-        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -183,7 +183,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             metalist=metalist[1:]
 
         while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters'):
+            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status'):
                 pass
             elif  metalist[0].startswith('Updated'):
                 self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -18,6 +18,7 @@
 import time
 import logging
 import re
+import urllib
 import urllib2
 
 from .. import BeautifulSoup as bs
@@ -69,6 +70,41 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
+                or 'There is no such account on our website' in data \
+                or "Noch kein registrierter Benutzer?" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self,url):
+        params = {}
+
+        if self.password:
+            params['nickname'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['nickname'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['savelogindata'] = '1'
+        params['a'] = 'l'
+        params['submit'] = 'Login...'
+
+        loginUrl = 'https://ssl.fanfiktion.de/'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['nickname']))
+        d = self._postUrl(loginUrl,params)
+    
+        if "Login erfolgreich" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['nickname']))
+            raise exceptions.FailedToLogin(url,params['nickname'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -83,6 +119,14 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Uhr ist diese Geschichte nur nach einer" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -140,6 +184,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5) ## ffde has "floodlock" protection
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NationalLibraryNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NationalLibraryNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ntlb')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","West Wing")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'national-library.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.national-library.net','national-library.net']
+
+    def getSiteExampleURLs(self):
+        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'ncisfic.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfic.com','ncisfic.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                if "No Series" not in value.nextSibling.string:
+                    self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+        
+        a=story.findNext('font')
+        if 'Complete' in a.string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -190,12 +190,19 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        for a in soup.findAll('table'):
-            a.extract()
+                                     selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.
 
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
+        # Ugh.  chapter html doesn't haven't anything useful around it to demarcate.
+        for a in soup.findAll('table'):
+            a.extract()        
+
+        for a in soup.findAll('head'):
+            a.extract()
+
+        html = soup.find('html')
+        html.name='div'
+            
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/base_adapter.py

@@ -160,9 +160,9 @@ class BaseSiteAdapter(Configurable):
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url,urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
         else:
-            return self.opener.open(url).read()
+            return self.opener.open(url.replace(' ','%20')).read()
     
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
@@ -235,6 +235,10 @@ class BaseSiteAdapter(Configurable):
     def getStoryMetadataOnly(self):
         if not self.metadataDone:
             self.extractChapterUrlsAndMetadata()
+            
+            if not self.story.getMetadataRaw('dateUpdated'):
+                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+
             self.metadataDone = True
         return self.story
 

fanficdownloader/configurable.py

@@ -54,6 +54,17 @@ class Configurable(object):
     def addConfigSection(self,section):
         self.sectionslist.insert(0,section)
 
+    def hasConfig(self, key):
+        for section in self.sectionslist:
+            try:
+                self.config.get(section,key)
+                #print("found %s in section [%s]"%(key,section))
+                return True
+            except:
+                pass
+
+        return False
+        
     def getConfig(self, key):
         val = ""
         for section in self.sectionslist:

fanficdownloader/writers/base_writer.py

@@ -149,18 +149,22 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                        if self.getConfigList(entry):
+                            
+                        if self.hasConfig(entry+"_label"):
                             label=self.getConfig(entry+"_label")
                         else:
+                            print("Using fallback label for %s_label"%entry)
                             label=self.titleLabels[entry]
 
                         # If the label for the title entry is empty, use the
-                        # 'no title' option if there is one.  
+                        # 'no title' option if there is one.
                         if label == "" and NO_TITLE_ENTRY:
                            TEMPLATE= NO_TITLE_ENTRY
                            
                         self._write(out,TEMPLATE.substitute({'label':label,
                                                              'value':self.story.getMetadata(entry)}))
+                else:
+                    self._write(out, entry)
 
             self._write(out,END.substitute(self.story.metadata))
 

index.html

@@ -54,18 +54,27 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>fanfiction.net Fixed</h3>
-	  <p>
-
-	    Fixes are in place now for the latest round of fanfiction.net changes (as of Jun 28, 2012).
-	    
-	  </p>
 	  <h3>New Sites</h3>
 	  <p>
 	    
-	    New sites www.fanfiktion.de, ponyfictionarchive.net and sg1-heliopolis.com added.
+	    New sites ncisfic.com and national-library.net added.
 	    <br />Thanks, Ida!
 	    
+	  </p>
+	  <h3>New Fixes</h3>
+	  <p>
+
+	    Fixes to support www.fanfiktion.de and archiveofourown.org username and password.
+	    
+	  </p>
+	  <p>
+
+	    <b>Note:</b> As part of the archiveofourown.org fix, the
+	    configuration section for AO3 is now
+	    [archiveofourown.org], not [www.archiveofourown.org].
+	    Please change your personal configuration accordingly, if
+	    you have customized your settings for archiveofourown.org.
+	    
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -75,7 +84,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-16.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-17.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -446,6 +455,16 @@
 	    <br />Or:
 	    <br /><a href="http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234">http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>ncisfic.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://ncisfic.com/viewstory.php?storyid=1234">http://ncisfic.com/viewstory.php?storyid=1234</a>
+	  </dd>
+	  <dt>national-library.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://national-library.net/viewstory.php?storyid=1234">http://national-library.net/viewstory.php?storyid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -71,7 +71,9 @@ extratags_label:Extra Tags
 version_label:FFDL Version
 
 ## items to include in the title page
-## Empty entries will *not* appear, even if in the list.
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
 titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
@@ -297,6 +299,19 @@ extratags: FanFiction,Testing,HTML
 
 [archive.skyehawke.com]
 
+[archiveofourown.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -384,6 +399,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[national-library.net]
+
+[ncisfic.com]
+
 [nfacommunity.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -450,12 +469,6 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [www.thealphagate.com]
 
-[www.archiveofourown.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.checkmated.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -478,6 +491,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 never_make_cover: true
 
 [www.fanfiktion.de]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
 [www.ficbook.net]