Update index.html, plugin-defaults.ini, bump versions.

Add DB error to mw.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
-application: ffd-retief-hrd
-version: 4-3-3
+application: fanfictiondownloader
+version: 4-4-4
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 2)
+    version             = (1, 5, 10)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/about.txt

@@ -16,5 +16,13 @@ The official distro channel for this plugin is there: <a href="http://www.mobile
 group</a> for the downloader.  That covers the web application and CLI, too.
 </p>
 
-The source for this plugin is available
-<a href="http://code.google.com/p/fanficdownloader/source/checkout">here</a>.
+The source for this plugin is available at it's 
+<a href="http://code.google.com/p/fanficdownloader">project home</a>.
+<hr />
+
+<p>
+See the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">list of supported sites</a>.
+</p>
+<p>
+Read the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
+</p>

calibre-plugin/config.py

@@ -35,6 +35,7 @@ all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 # Set defaults used by all.  Library specific settings continue to
 # take from here.
 all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
+
 all_prefs.defaults['updatemeta'] = True
 all_prefs.defaults['updatecover'] = False
 all_prefs.defaults['keeptags'] = False
@@ -43,12 +44,15 @@ all_prefs.defaults['updatedefault'] = True
 all_prefs.defaults['fileform'] = 'epub'
 all_prefs.defaults['collision'] = OVERWRITE
 all_prefs.defaults['deleteotherforms'] = False
-all_prefs.defaults['adddialogstatsontop'] = False
+all_prefs.defaults['adddialogstaysontop'] = False
+all_prefs.defaults['includeimages'] = False
+
 all_prefs.defaults['send_lists'] = ''
 all_prefs.defaults['read_lists'] = ''
 all_prefs.defaults['addtolists'] = False
 all_prefs.defaults['addtoreadlists'] = False
 all_prefs.defaults['addtolistsonread'] = False
+
 all_prefs.defaults['custom_cols'] = {}
 
 # The list of settings to copy from all_prefs or the previous library
@@ -62,7 +66,8 @@ copylist = ['personal.ini',
             'fileform',
             'collision',
             'deleteotherforms',
-            'adddialogstatsontop']
+            'adddialogstaysontop',
+            'includeimages']
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
@@ -121,6 +126,10 @@ class ConfigWidget(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label.setOpenExternalLinks(True)
+        self.l.addWidget(label)
+
         tab_widget = QTabWidget(self)
         self.l.addWidget(tab_widget)
 
@@ -153,7 +162,8 @@ class ConfigWidget(QWidget):
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
-        prefs['adddialogstatsontop'] = self.basic_tab.adddialogstatsontop.isChecked()
+        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
+        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
 
         if self.list_tab:
             # lists
@@ -266,10 +276,16 @@ class BasicTab(QWidget):
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.adddialogstatsontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
-        self.adddialogstatsontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
-        self.adddialogstatsontop.setChecked(prefs['adddialogstatsontop'])
-        self.l.addWidget(self.adddialogstatsontop)
+        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
+        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
+        self.l.addWidget(self.adddialogstaysontop)
+
+        # this is a cheat to make it easier for users to realize there's a new include_images features.
+        self.includeimages = QCheckBox("Include images in EPUBs?",self)
+        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[defaults]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages.setChecked(prefs['includeimages'])
+        self.l.addWidget(self.includeimages)
         
         self.l.insertStretch(-1)
         

calibre-plugin/dialogs.py

@@ -71,7 +71,7 @@ class AddNewDialog(SizePersistedDialog):
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
 
-        if prefs['adddialogstatsontop']:
+        if prefs['adddialogstaysontop']:
             QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
         
         self.setMinimumWidth(300)
@@ -159,25 +159,40 @@ class AddNewDialog(SizePersistedDialog):
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
-        
+
+
+class FakeLineEdit():
+    def __init__(self):
+        pass
+    
+    def text(self):
+        pass
+    
 class UserPassDialog(QDialog):
     '''
     Need to collect User/Pass for some sites.
     '''
-    def __init__(self, gui, site):
+    def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
         self.gui = gui
         self.status=False
-        self.setWindowTitle('User/Password')
 
         self.l = QGridLayout()
         self.setLayout(self.l)
 
-        self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+        if exception.passwdonly:
+            self.setWindowTitle('Password')
+            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            # user isn't used, but it's easier to still have it for
+            # post processing.
+            self.user = FakeLineEdit()
+        else:
+            self.setWindowTitle('User/Password')
+            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
         
-        self.l.addWidget(QLabel("User:"),1,0)
-        self.user = QLineEdit(self)
-        self.l.addWidget(self.user,1,1)
+            self.l.addWidget(QLabel("User:"),1,0)
+            self.user = QLineEdit(self)
+            self.l.addWidget(self.user,1,1)
    
         self.l.addWidget(QLabel("Password:"),2,0)
         self.passwd = QLineEdit(self)

calibre-plugin/ffdl_plugin.py

@@ -220,7 +220,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         text = get_resources('about.txt')
         AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
-        
+
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                            shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                            unique_name=None):
@@ -413,15 +413,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         adapter = adapters.getAdapter(ffdlconfig,url,fileform)
 
         options['personal.ini'] = prefs['personal.ini']
+        if prefs['includeimages']:
+            # this is a cheat to make it easier for users.
+            options['personal.ini'] = '''[defaults]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + options['personal.ini']
 
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
         for x in range(0,2):
             try:
                 adapter.getStoryMetadataOnly()
-            except exceptions.FailedToLogin:
+            except exceptions.FailedToLogin, f:
                 print("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url)
+                userpass = UserPassDialog(self.gui,url,f)
                 userpass.exec_() # exec_ will make it act modal
                 if userpass.status:
                     adapter.username = userpass.user.text()
@@ -441,7 +448,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
-        book['tags'] = writer.getTags()
+        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
         book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
         book['series'] = story.getMetadata("series")
         
@@ -499,8 +506,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if len(identicalbooks) > 1:
                 raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
 
+            ## changed: add new book when CALIBREONLY if none found.
             if collision == CALIBREONLY and not identicalbooks:
-                raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
+                collision = ADDNEW
+                options['collision'] = ADDNEW
+            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
 
             if len(identicalbooks)>0:
                 book_id = identicalbooks.pop()
@@ -508,7 +518,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 book['icon'] = 'edit-redo.png'
 
         if book_id != None and collision != ADDNEW:
-            if options['collision'] in (CALIBREONLY):
+            if collision in (CALIBREONLY):
                 book['comment'] = 'Metadata collected.'
                 # don't need temp file created below.
                 return
@@ -633,12 +643,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def _update_books_completed(self, book_list, options={}):
         
         add_list = filter(lambda x : x['good'] and x['added'], book_list)
+        add_ids = [ x['calibre_id'] for x in add_list ]
         update_list = filter(lambda x : x['good'] and not x['added'], book_list)
         update_ids = [ x['calibre_id'] for x in update_list ]
         
         if len(add_list):
             ## even shows up added to searchs.  Nice.
             self.gui.library_view.model().books_added(len(add_list))
+            self.gui.library_view.model().refresh_ids(add_ids)
 
         if update_ids:
             self.gui.library_view.model().refresh_ids(update_ids)

defaults.ini

@@ -112,9 +112,9 @@ zip_output: false
 ## Can include directories.  .zip will be added if not in name somewhere
 zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
-## Normally, try to make the output file name 'safe' by removing
-## invalid filename chars.  Applies to both output_filename &
-## zip_filename.  
+## Normally, try to make the filenames 'safe' by removing invalid
+## filename chars.  Applies to default_cover_image, output_filename &
+## zip_filename.
 allow_unsafe_filename: false
 
 ## entries to make epub subjects and calibre tags
@@ -147,6 +147,11 @@ extratags: FanFiction
 # (.*)Great(.*)=>\1Moderate\2
 # .*-Centered=>
 
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -244,15 +249,28 @@ output_css:
 ## in chapters.
 #make_firstimage_cover: false
 
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
 ## If set, and there isn't already a cover image from the adapter or
 ## from make_firstimage_cover, this image will be made the cover.
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
 ## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
 #default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
 #default_cover_image:http://www.somesite.com/someimage.gif
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 ## Resize images down to width, height, preserving aspect ratio.
 ## Nook size, with margin.
 image_max_size: 580, 725
@@ -261,6 +279,13 @@ image_max_size: 580, 725
 ## space.
 #grayscale_images: false
 
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
@@ -281,8 +306,95 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML
 
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.mugglenet.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.archiveofourown.org]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fanfiction.net]
 
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fictionally.org storyIds are not unique.  Combine with authorId.
+output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
+
 [www.fictionpress.com]
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
@@ -295,6 +407,74 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.mediaminer.org]
+
+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+[www.potionsandsnitches.net]
+
+[www.siye.co.uk]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -319,71 +499,6 @@ collect_series: false
 
 [www.whofic.com]
 
-[www.mediaminer.org]
-
-[www.thewriterscoffeeshop.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
-collect_series: false
-
-[www.ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fictionalley.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## fictionally.org storyIds are not unique.  Combine with authorId.
-output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
-
-[www.harrypotterfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fimfiction.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.tthfanfic.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-## tth is a little unusual--it doesn't require user/pass, but the site
-## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
-#username:YourName
-#password:yourpassword
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -63,7 +63,7 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.  Not compatible with inserted TOC.",)
+                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
                      help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
@@ -137,10 +137,13 @@ def main():
        for x in range(0,2):
            try:
                adapter.getStoryMetadataOnly()
-           except exceptions.FailedToLogin:
-               print "Login Failed, Need Username/Password."
-               sys.stdout.write("Username: ")
-               adapter.username = sys.stdin.readline().strip()
+           except exceptions.FailedToLogin, f:
+               if f.passwdonly:
+                   print "Story requires a password."
+               else:
+                   print "Login Failed, Need Username/Password."
+                   sys.stdout.write("Username: ")
+                   adapter.username = sys.stdin.readline().strip()
                adapter.password = getpass.getpass(prompt='Password: ')
                #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
            except exceptions.AdultCheckRequired:

fanficdownloader/adapters/__init__.py

@@ -27,7 +27,6 @@ from .. import exceptions as exceptions
 import adapter_test1
 import adapter_fanfictionnet
 import adapter_castlefansorg
-import adapter_fanfictionnet
 import adapter_fictionalleyorg
 import adapter_fictionpresscom
 import adapter_ficwadcom
@@ -45,7 +44,13 @@ import adapter_whoficcom
 import adapter_siyecouk
 import adapter_archiveofourownorg
 import adapter_ficbooknet
-import adapter_gayauthorsorg
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -74,6 +74,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
         self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -83,9 +83,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
+
+        if "/images/missing_story.png" in data:
+            raise exceptions.StoryDoesNotExist(self.url)
         
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
+
+        if self.password:
+            params = {}
+            params['password'] = self.password
+            data = self._postUrl(self.url,params)
+
+        if "Enter the password the author set for this story to view it." in data:
+            if self.getConfig('fail_on_password'):
+                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
+            else:
+                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
 
@@ -102,7 +116,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         for chapter in soup.findAll("a", {"class":"chapter_link"}):
             chapterDates.append(chapter.span.extract().text.strip("()"))
             self.chapterUrls.append((chapter.text.strip(), "http://"+self.getSiteDomain() + chapter['href']))
-        
+
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
@@ -157,21 +171,25 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
         now = datetime.datetime.now()
         
-        # Get the date of creation from the first chapter
-        datePublished_text = chapterDates[0]
-        day, month = datePublished_text.split()
-        day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
-        if datePublished > now :
-            datePublished = datePublished.replace(year=now.year-1)
-        self.story.setMetadata("datePublished", datePublished)
         dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
         dateUpdated_soup.find('span').extract()
         dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
         if dateUpdated > now :
-            dateUpdated = datePublished.replace(year=now.year-1)
+            dateUpdated = dateUpdated.replace(year=now.year-1)
         self.story.setMetadata("dateUpdated", dateUpdated)
         
+        # Get the date of creation from the first chapter
+        if len(chapterDates) > 0:
+            datePublished_text = chapterDates[0]
+            day, month = datePublished_text.split()
+            day = re.sub(r"[^\d.]+", '', day)
+            datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
+            if datePublished > now :
+                datePublished = datePublished.replace(year=now.year-1)
+            self.story.setMetadata("datePublished", datePublished)
+        else:
+            self.story.setMetadata("datePublished", dateUpdated)
+            
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})

fanficdownloader/adapters/adapter_gayauthorsorg.py

@@ -1,205 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import logging
-import re
-import urllib2
-from urllib import unquote
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, makeDate
-
-def getClass():
-    return GayAuthorsAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GayAuthorsAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-       
-		
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        # unqoute, change '_' and ' ' to '-', downcase, and remove non-[a-z0-9-]
-        authid = unquote(self.parsedUrl.path.split('/',)[2])
-        authid = authid.lower().replace('_','-').replace(' ','-')
-        authid = re.sub(r"[^a-z0-9-]","",authid)
-        
-        self.story.setMetadata('authorId',authid)
-        logging.debug("authorId: (%s)"%self.story.getMetadata('authorId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('authorId') + '/' + self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ga')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.gayauthors.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/author/storytitle"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story/")+r".*?/\w+.*?$"
-
-    
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-
-        url = self.url
-        logging.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        msoup = soup.find('div', {'class' : 'story'})
-        if msoup == None:
-            msoup = soup.find('div', {'class' : 'story ispinned'})
-        csoup = soup.find('div', {'id' : 'story_chapters'})
-        
-        ## Title
-        a = msoup.find('span', {'class' : 'title'})
-        title=a.find('span', {'itemprop' : 'name'})
-        self.story.setMetadata('title',title.text)
-        try:
-            # Find Series name from series URL.
-            series = a.find('span',{'class':"description"})
-            series_name = series.find('a')
-            series_name.extract()
-            series_index = int(series.text.split(' ')[1])
-            self.setSeries(series_name.text, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-        
-        # Find authorid and URL from... author url.
-        a = msoup.find('a', href=re.compile(r'/author/'+self.story.getMetadata('authorId')))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',a.text)
-		
-
-        # Find the chapters:
-        spans=csoup.findAll('span', {'class' : 'desc chapter-info'})
-        for span in spans:
-            span.extract()
-        for chapter in csoup.findAll('a'):
-            # just in case there's tags, like <i> in chapter titles.
-            a=chapter['href'].split(self.story.getMetadata('author'))
-            a=a[0]+self.story.getMetadata('authorId')+a[1]
-            self.chapterUrls.append((stripHTML(chapter),a))
-
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-		
-        cats = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_0__category\w+$'))
-        for cat in cats:
-            self.story.addToList('category',cat.text)
-		
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_1__genre\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_2__tag\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-			
-        status = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_3__status\w+$'))
-        self.story.setMetadata('status',status.text)
-		
-        rating = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_4__rating\w+$'))
-        self.story.setMetadata('rating',rating.text)
-		
-        summary = msoup.find('span', {'itemprop' : 'description'})
-        self.setDescription(self.url,summary.text)
-        #self.story.setMetadata('description',summary.text)
-	
-
-        stats = msoup.find('dl',{'class':'info'})
-        dt = stats.findAll('dt')
-        dd = stats.findAll('dd')
-        for x in range(0,len(dt)):
-            label = dt[x].text
-            value = dd[x].text
-
-            if 'Words:' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Published:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-				
-            if 'Updated:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logging.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        data = data[data.index("<div id='chapter-content'>"):]
-        soup = bs.BeautifulSoup(data) # this one's happier with Soup, not StoneSoup for some reason.
-        
-        div = soup.find('div', {'id' : 'chapter-content'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return HPFandomNetAdapterAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/eff/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpfdm') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfandom.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/eff/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authdata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        # fix a typo in the site HTML so I can find the Characters list.
+        authdata = authdata.replace('<td width=10%">','<td width="10%">')
+
+        # hpfandom.net only seems to indicate adult-only by javascript on the story/chapter links.
+        if "javascript:if (confirm('Slash/het fiction which incorporates sexual situations to a somewhat graphic degree and some violence. ')) location = 'viewstory.php?sid=%s'"%self.story.getMetadata('storyId') in authdata \
+                and not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        authsoup = bs.BeautifulSoup(authdata)
+
+        reviewsa = authsoup.find('a', href="reviews.php?sid="+self.story.getMetadata('storyId')+"&a=")
+        # <table><tr><td><p><b><a ...>
+        metablock = reviewsa.findParent("table")
+        #print("metablock:%s"%metablock)
+        
+        ## Title
+        titlea = metablock.find('a', href=re.compile("viewstory.php"))
+        #print("titlea:%s"%titlea)
+        if titlea == None:
+            raise exceptions.FailedToDownload("Story URL (%s) not found on author's page, can't use chapter URLs"%url)
+        self.story.setMetadata('title',stripHTML(titlea))
+        
+        # Find the chapters: !!! hpfandom.net differs from every other
+        # eFiction site--the sid on viewstory for chapters is
+        # *different* for each chapter
+        for chapter in soup.findAll('a', {'href':re.compile(r"viewstory.php\?sid=\d+&i=\d+")}):
+            m = re.match(r'.*?(viewstory.php\?sid=\d+&i=\d+).*?',chapter['href'])
+            # just in case there's tags, like <i> in chapter titles.
+            #print("====chapter===%s"%m.group(1))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eff/'+m.group(1)))
+
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        summary = metablock.find("td",{"class":"summary"})
+        summary.name='span'
+        self.setDescription(url,summary)
+
+        # words & completed in first row of metablock.
+        firstrow = stripHTML(metablock.find('tr'))
+        # A Mother's Love xx Going Grey 1 (G+) by Kiristeen | Reviews - 18 | Words: 27468 | Completed: Yes
+        m = re.match(r".*?\((?P<rating>[^)]+)\).*?Words: (?P<words>\d+).*?Completed: (?P<status>Yes|No)",firstrow)
+        if m != None:
+            if m.group('rating') != None:
+                self.story.setMetadata('rating', m.group('rating'))
+                
+            if m.group('words') != None:
+                self.story.setMetadata('numWords', m.group('words'))
+
+            if m.group('status') != None:
+                if 'Yes' in m.group('status'):
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                
+
+        # <tr><td width="10%" valign="top">Chapters:</td><td width="40%" valign="top">4</td>
+        # <td width="10%" valign="top">Published:</td><td width="40%" valign="top">2010.09.29</td></tr>
+        # <tr><td width="10%" valign="top">Completed:</td><td width="40%" valign="top">Yes</td><td width="10%" valign="top">Updated:</td><td width="40%" valign="top">2010.10.03</td></tr>
+        labels = metablock.findAll('td',{'width':'10%'})
+        for td in labels:
+            label = td.string
+            value = td.nextSibling.string
+            #print("\nlabel:%s\nvalue:%s\n"%(label,value))
+
+            if 'Category' in label:
+                cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.split(','):
+                    self.story.addToList('characters',char.strip())
+
+            if 'Genre' in label:
+                for genre in value.split(','):
+                    self.story.addToList('genre',genre.strip())
+
+            if 'Warnings' in label:
+                for warning in value.split(','):
+                    if warning.strip() != 'none':
+                        self.story.addToList('warnings',warning.strip())
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        # There's no good wrapper around the chapter text. :-/
+        # There are, however, tables with width=100% just above and below the real text.
+        data = re.sub(r'<table width="100%">.*?</table>','<div name="storybody">',
+                      data,count=1,flags=re.DOTALL)
+        
+        data = re.sub(r'<table width="100%">.*?</table>','</div>',
+                      data,count=1,flags=re.DOTALL)
+        
+        soup = bs.BeautifulStoneSoup(data,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        div = soup.find("div",{'name':'storybody'})
+        #print("\n\ndiv:%s\n\n"%div)
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return KSArchiveComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class KSArchiveComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ksa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Star Trek") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ksarchive.com','ksarchive.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ksarchive.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=2" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    if cat.string.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                self.story.addToList('characters','Kirk')
+                self.story.addToList('characters','Spock')
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## In addition to Genre (which is very site specific) KSA
+            ## has 'Story Type', which is much more what most sites
+            ## call genre.
+            if 'Story Type' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MidnightwhispersCaAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mw') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Queer as Folk") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.midnightwhispers.ca' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -0,0 +1,331 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MuggleNetComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MuggleNetComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mgln') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.mugglenet.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if "class='errortext'>Registered Users Only" in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            # http://fanfiction.mugglenet.com/viewstory.php?sid=91079&ageconsent=ok&warning=3
+            addurl = "&ageconsent=ok&warning=3" # XXX  &warning=5
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            #print("\nurl:%s\ndata:\n%s\n"%(url,data))
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=%s((?:&ageconsent=ok)?&warning=\d+)'"%self.story.getMetadata('storyId'),data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+        # Not good enough-- content can contain a ('), which ends the content prematurely.
+        # metadesc = soup.find('meta',{'name':'description'})
+        # print("removeAllEntities(metadesc['content']):\n%s\n"%removeAllEntities(metadesc['content']))
+        start='<span class="label">Summary: </span>'
+        end='<span class="label">Rated:</span>'
+        summarydata = data[data.index(start)+len(start):data.index(end)]
+        #print("summarydata:\n%s\n"%summarydata)
+        self.setDescription(url,bs.BeautifulSoup(summarydata))
+        
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            # not good enough--poorly formated summary html will break it.
+            # if 'Summary' in label:
+            #     ## Everything until the next span class='label'
+            #     svalue = ""
+            #     while not defaultGetattr(value,'class') == 'label':
+            #         svalue += str(value)
+            #         value = value.nextSibling
+            #     self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return NfaCommunityComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nfa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.nfacommunity.com','nfacommunity.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nfacommunity.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  nfacommunity has two: 4 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PortkeyOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','prtky') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.portkey.org' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        # portkey screws around with using a different URL to set the
+        # cookie and it's a pain.  So... cheat!
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='verify17', value='1',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if "You must be over 18 years of age to view it" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/profile/\d+"))
+        #print("======a:%s"%a)
+        self.story.setMetadata('authorId',a['href'].split('/')[-1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'/story/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(titlea))
+        metablock = titlea.parent
+        
+        # Find the chapters:
+        for chapter in soup.find('select',{'name':'select5'}).findAll('option', {'value':re.compile(r'/story/'+self.story.getMetadata('storyId')+"/\d+$")}):
+            # just in case there's tags, like <i> in chapter titles.
+            chtitle = stripHTML(chapter)
+            if not chtitle:
+                chtitle = "(Untitled Chapter)"
+            self.chapterUrls.append((chtitle,'http://'+self.host+chapter['value']))
+            
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+        # <SPAN class="dark-small-bold">Contents:</SPAN> <SPAN class="small-grey">NC17 </SPAN> 
+        # <SPAN class="dark-small-bold">Published: </SPAN><SPAN class="small-grey">12/11/07</SPAN>
+        # <SPAN class="dark-small-bold"><BR>
+        # Description:</SPAN> <SPAN class="small-black">A special book helps Harry tap into the power the Dark Lord knows not.  Of course it’s a book on sex magic and rituals… but Harry’s not complaining.  Spurned on by the ghost of a pervert founder, Harry leads his friends in the hunt for Voldemort’s Horcruxes.
+        # EROTIC COMEDY!  Loads of crude humor and sexual situations!
+        # </SPAN>
+        labels = metablock.findAll('span',{'class':'dark-small-bold'})
+        for labelspan in labels:
+            value = labelspan.findNext('span').string
+            label = stripHTML(labelspan)
+#            print("\nlabel:%s\nlabel:%s\nvalue:%s\n"%(labelspan,label,value))
+
+            if 'Description' in label:
+                self.setDescription(url,value)
+
+            if 'Contents' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            # if 'Categories' in label:
+            #     cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+            #     catstext = [cat.string for cat in cats]
+            #     for cat in catstext:
+            #         self.story.addToList('category',cat.string)
+
+            # if 'Characters' in label:
+            #     chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+            #     charstext = [char.string for char in chars]
+            #     for char in charstext:
+            #         self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                # genre is typo'ed on the site--it falls between the
+                # dark-small-bold label and dark-small-bold content
+                # spans.
+                svalue = ""
+                value = labelspan.nextSibling
+                while not defaultGetattr(value,'class') == 'dark-small-bold':
+                    svalue += str(value)
+                    value = value.nextSibling
+                    
+                for genre in svalue.split("/"):
+                    genre = genre.strip()
+                    if genre != 'None':
+                        self.story.addToList('genre',genre)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            # if 'Warnings' in label:
+            #     warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+            #     warningstext = [warning.string for warning in warnings]
+            #     self.warning = ', '.join(warningstext)
+            #     for warning in warningstext:
+            #         self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'Completed' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        # try:
+        #     # Find Series name from series URL.
+        #     a = metablock.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+        #     series_name = a.string
+        #     series_url = 'http://'+self.host+'/'+a['href']
+
+        #     # use BeautifulSoup HTML parser to make everything easier to find.
+        #     seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+        #     storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+        #     i=1
+        #     for a in storyas:
+        #         if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+        #             self.setSeries(series_name, i)
+        #             break
+        #         i+=1            
+        # except:
+        #     # I find it hard to care if the series parsing fails
+        #     pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        #print("soup:%s"%soup)
+        tag = soup.find('td', {'class' : 'story'})
+        if tag == None and "<center><b>Chapter does not exist!</b></center>" in data:
+            print("Chapter is missing at: %s"%url)
+            return  self.utf8FromSoup(url,bs.BeautifulStoneSoup("<div><p><center><b>Chapter does not exist!</b></center></p><p>Chapter is missing at: <a href='%s'>%s</a></p></div>"%(url,url)))
+        tag.name='div' # force to be a div to avoid problems with nook.
+
+        centers = tag.findAll('center')
+        # first two and last two center tags are some script, 'report
+        # story', 'report story' and an ad.
+        centers[0].extract()
+        centers[1].extract()
+        centers[-1].extract()
+        centers[-2].extract()
+
+        if None == tag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,tag)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -61,14 +61,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
 
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
     def extractChapterUrlsAndMetadata(self):
 
         url = self.url+'&index=1'

fanficdownloader/adapters/adapter_siyecouk.py

@@ -45,9 +45,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        # self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        # self.password = ""
-        # self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
@@ -83,53 +80,9 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
 
-    # ## Login seems to be reasonably standard across eFiction sites.
-    # def needToLoginCheck(self, data):
-    #     if 'Registered Users Only' in data \
-    #             or 'There is no such account on our website' in data \
-    #             or "That password doesn't match the one in our database" in data:
-    #         return True
-    #     else:
-    #         return False
-        
-    # def performLogin(self, url):
-    #     params = {}
-
-    #     if self.password:
-    #         params['penname'] = self.username
-    #         params['password'] = self.password
-    #     else:
-    #         params['penname'] = self.getConfig("username")
-    #         params['password'] = self.getConfig("password")
-    #     params['cookiecheck'] = '1'
-    #     params['submit'] = 'Submit'
-    
-    #     loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-    #     logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-    #                                                           params['penname']))
-    
-    #     d = self._fetchUrl(loginUrl, params)
-    
-    #     if "Member Account" not in d : #Member Account
-    #         logging.info("Failed to login to URL %s as %s" % (loginUrl,
-    #                                                           params['penname']))
-    #         raise exceptions.FailedToLogin(url,params['penname'])
-    #         return False
-    #     else:
-    #         return True
-
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # if self.is_adult or self.getConfig("is_adult"):
-        #     # Weirdly, different sites use different warning numbers.
-        #     # If the title search below fails, there's a good chance
-        #     # you need a different number.  print data at that point
-        #     # and see what the 'click here to continue' url says.
-        #     addurl = "&ageconsent=ok&warning=4" # XXX
-        # else:
-        #     addurl=""
-
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
         # Except it doesn't this time. :-/
@@ -144,20 +97,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             else:
                 raise e
 
-        # if self.needToLoginCheck(data):
-        #     # need to log in for this one.
-        #     self.performLogin(url)
-        #     data = self._fetchUrl(url)
-
-        # # The actual text that is used to announce you need to be an
-        # # adult varies from site to site.  Again, print data before
-        # # the title search to troubleshoot.
-        # if "Age Consent Required" in data: # XXX 
-        #     raise exceptions.AdultCheckRequired(self.url)
-            
-        # if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-        #     raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data

fanficdownloader/adapters/adapter_test1.py

@@ -186,7 +186,7 @@ br breaks<br><br>
 br breaks<br><br>
 <hr>
 horizontal rules
-<hr>
+<hr size=1 noshade>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheQuidditchPitchOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the  part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tqdpch') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thequidditchpitch.org' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thequidditchpitch.org','thequidditchpitch.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only - Not suitable for readers under the age of legal consent in their country.' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if ("Not suitable for readers under the age of legal consent in their country." in data \
+                or "Not suitable for readers under 16 yrs. \r\nStories may contain violence, slight nudity, and/or sexual situations." in data ) \
+                and not (self.is_adult or self.getConfig("is_adult")): # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # span?  Really?  span?  Yeah... I don't think so.
+        div = soup.find('span', {'style' : 'font-size: 100%;'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -179,14 +179,15 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         verticaltable = soup.find('table', {'class':'verticaltable'})
 
         BtVS = True
+        BtVSNonX = False
         for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
             if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                 self.story.addToList('category',cat.string)
             else:
                 if 'Non-BtVS' in cat.string:
                     BtVS = False
-        if BtVS:
-            self.story.addToList('category','Buffy: The Vampire Slayer')
+                if 'BtVS/AtS Non-Crossover' == cat.string:
+                    BtVSNonX = True
 
         verticaltabletds = verticaltable.findAll('td')
         self.story.setMetadata('rating', verticaltabletds[2].string)
@@ -204,6 +205,13 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
             if( icon['title'] not in ['Non-Crossover'] ) :
                 self.story.addToList('genre',icon['title'])
+            else:
+                if not BtVSNonX:
+                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
+
+        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        if BtVS:
+            self.story.addToList('category','Buffy: The Vampire Slayer')
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapnav' } )

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -38,6 +38,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
         self.story.addToList("category","Twilight")
+        self.is_adult = False
         self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
         self.password = ""
         
@@ -99,7 +100,16 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
     def extractChapterUrlsAndMetadata(self):
 
-        url = self.url+'&index=1'
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1" # XXX
+        else:
+            addurl=""
+            
+        url = self.url+'&index=1'+addurl
         logging.debug("URL: "+url)
 
         try:
@@ -118,6 +128,9 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
 
+        if "Contains Explicit Content for mature adults only! May contain graphic violence, mature sexual situations, and explicit language. Read with caution." in data:
+            raise exceptions.AdultCheckRequired(self.url)
+        
         # problems with some stories, but only in calibre.  I suspect
         # issues with different SGML parsers in python.  This is a
         # nasty hack, but it works.
@@ -139,7 +152,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 

fanficdownloader/adapters/base_adapter.py

@@ -215,7 +215,9 @@ class BaseSiteAdapter(Configurable):
                     self.getConfig('default_cover_image'):
                 self.story.addImgUrl(self,
                                      None,
-                                     self.getConfig('default_cover_image'),
+                                     #self.getConfig('default_cover_image'),
+                                     self.story.formatFileName(self.getConfig('default_cover_image'),
+                                                               self.getConfig('allow_unsafe_filename')),
                                      self._fetchUrlRaw,
                                      cover=True)
         return self.story
@@ -272,8 +274,9 @@ class BaseSiteAdapter(Configurable):
             self.story.setMetadata('description',stripHTML(svalue))
         #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n")
 
-    # this gives us a unicode object, not just a string containing bytes.
+    # This gives us a unicode object, not just a string containing bytes.
     # (I gave soup a unicode string, you'd think it could give it back...)
+    # Now also does a bunch of other common processing for us.
     def utf8FromSoup(self,url,soup,fetch=None):
         if not fetch:
             fetch=self._fetchUrlRaw
@@ -283,8 +286,11 @@ class BaseSiteAdapter(Configurable):
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))
             for img in soup.findAll('img'):
-                img['longdesc']=img['src']
-                img['src']=self.story.addImgUrl(self,url,img['src'],fetch)
+                # some pre-existing epubs have img tags that had src stripped off.
+                if img.has_key('src'):
+                    img['longdesc']=img['src']
+                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch,
+                                                    coverexclusion=self.getConfig('cover_exclusion_regexp'))
 
         for attr in soup._getAttrMap().keys():
             if attr not in acceptable_attributes:
@@ -294,9 +300,9 @@ class BaseSiteAdapter(Configurable):
             for attr in t._getAttrMap().keys():
                 if attr not in acceptable_attributes:
                     del t[attr] ## strip all tag attributes except href and name
-                    
+
             # these are not acceptable strict XHTML.  But we do already have 
-	    # CSS classes of the same names defined in constants.py
+	    # CSS classes of the same names defined
             if t.name in ('u'):
                 t['class']=t.name
                 t.name='span'
@@ -307,9 +313,23 @@ class BaseSiteAdapter(Configurable):
             if t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
                 
+        retval = soup.__str__('utf8').decode('utf-8')
+
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        if self.getConfig('nook_img_fix'):
+            # if the <img> tag doesn't have a div or a p around it,
+            # nook gets confused and displays it on every page after
+            # that under the text for the rest of the chapter.
+            retval = re.sub(r"(?!<(div|p)>)\s*(?P<imgtag><img[^>]+>)\s*(?!</(div|p)>)",
+                            "<div>\g<imgtag></div>",retval)
+            
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        return re.sub(r"</?body>\r?\n?","",soup.__str__('utf8').decode('utf-8'))
+        return re.sub(r"</?body>\r?\n?","",retval)
 
 fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",

fanficdownloader/epubutils.py

@@ -51,11 +51,13 @@ def get_update_data(inputio,
             # Hellmouth, which uses chapter0.html.
             if( item.getAttribute("media-type") == "application/xhtml+xml" ):
                 href=relpath+item.getAttribute("href")
-                print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
                 if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                     if getsoups:
                         soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
                         for img in soup.findAll('img'):
+                            newsrc=''
+                            longdesc=''
                             try:
                                 newsrc=get_path_part(href)+img['src']
                                 # remove all .. and the path part above it, if present.

fanficdownloader/exceptions.py

@@ -34,12 +34,16 @@ class InvalidStoryURL(Exception):
         return "Bad Story URL: (%s) for site: (%s) Example: (%s)" % (self.url, self.domain, self.example)
 
 class FailedToLogin(Exception):
-    def __init__(self,url,username):
+    def __init__(self,url, username, passwdonly=False):
         self.url=url
         self.username=username
+        self.passwdonly=passwdonly
         
     def __str__(self):
-        return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)
+        if self.passwdonly:
+            return "URL Failed, password required: (%s) " % (self.url)
+        else:
+            return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)
 
 class AdultCheckRequired(Exception):
     def __init__(self,url):

fanficdownloader/story.py

@@ -17,6 +17,7 @@
 
 import os, re
 import urlparse
+import string
 from math import floor
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
@@ -74,6 +75,9 @@ except:
                 export = True
 
             if normalize_format_name(img.format) != "jpg":
+                if img.mode == "P":
+                    # convert pallete gifs to RGB so jpg save doesn't fail.
+                    img = img.convert("RGB")
                 export = True
 
             if export:
@@ -187,7 +191,7 @@ class Story:
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.3'}
+            self.metadata = {'version':'4.4'}
         self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
@@ -275,9 +279,26 @@ class Story:
         "Chapters will be tuples of (title,html)"
         return self.chapters
 
+    def formatFileName(self,template,allowunsafefilename=True):
+        values = origvalues = self.getAllMetadata()
+        # fall back default:
+        if not template:
+            template="${title}-${siteabbrev}_${storyId}${formatext}"
+
+        if not allowunsafefilename:
+            values={}
+            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
+            for k in origvalues.keys():
+                values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
+
+        return string.Template(template).substitute(values).encode('utf8')
+
     # pass fetch in from adapter in case we need the cookies collected
     # as well as it's a base_story class method.
-    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False):
+    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
+        
+        url = url.strip() # ran across an image with a space in the
+                          # src. Browser handled it, so we'd better, too.
 
         # appengine (web version) isn't allowed to do images--just
         # gets too big too fast and breaks things.
@@ -295,11 +316,17 @@ class Story:
                      url,
                      '','',''))
             else:
+                toppath=""
+                if parsedUrl.path.endswith("/"):
+                    toppath = parsedUrl.path
+                else:
+                    toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,
-                     parsedUrl.path + url,
+                     toppath + '/' + url,
                      '','',''))
+                #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         prefix='ffdl'
         if imgurl not in self.imgurls:
@@ -311,11 +338,11 @@ class Story:
                                                 sizes,
                                                 configurable.getConfig('grayscale_images'))
             except Exception, e:
-                print("Failed to load image, skipping:\n%s\nException: %s"%(imgurl,e))
+                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
             
             # explicit cover, make the first image.
-            if cover:
+            if cover and not configurable.getConfig('never_make_cover'):
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                     # remove existing cover, if there is one.
                     del self.imgurls[0]
@@ -327,7 +354,14 @@ class Story:
             else:
                 self.imgurls.append(imgurl)
                 # First image, copy not link because calibre will replace with it's cover.
-                if (len(self.imgurls)==1 and configurable.getConfig('make_firstimage_cover')):
+                # Only if: No cover already AND
+                #          make_firstimage_cover AND
+                #          NOT never_make_cover AND
+                #          either no coverexclusion OR coverexclusion doesn't match
+                if self.cover == None and \
+                        configurable.getConfig('make_firstimage_cover') and \
+                        not configurable.getConfig('never_make_cover') and \
+                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})

fanficdownloader/writers/base_writer.py

@@ -18,7 +18,6 @@
 import re
 import os.path
 import datetime
-import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
@@ -123,24 +122,10 @@ class BaseStoryWriter(Configurable):
             return self.getBaseFileName()
 
     def getBaseFileName(self):
-        return self.formatFileName(self.getConfig('output_filename'))
+        return self.story.formatFileName(self.getConfig('output_filename'),self.getConfig('allow_unsafe_filename'))
     
     def getZipFileName(self):
-        return self.formatFileName(self.getConfig('zip_filename'))
-
-    def formatFileName(self,template):
-        values = origvalues = self.story.getAllMetadata()
-        # fall back default:
-        if not template:
-            template="${title}-${siteabbrev}_${storyId}${formatext}"
-
-        if not self.getConfig('allow_unsafe_filename'):
-            values={}
-            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
-            for k in origvalues.keys():
-                values[k]=re.sub(pattern,'_', removeAllEntities(self.story.getMetadata(k)))
-
-        return string.Template(template).substitute(values).encode('utf8')
+        return self.story.formatFileName(self.getConfig('zip_filename'),self.getConfig('allow_unsafe_filename'))
 
     def _write(self, out, text):
         out.write(text.encode('utf8'))

index.html

@@ -54,15 +54,19 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Site gayauthors.org</h3>
+	  <h3>New Site Added</h3>
 	  <p>
-	    Thanks to Ida Leter's hard work, we now support <a href="http://www.gayauthors.org">gayauthors.org</a>, a fanfiction site specializing in gay stories.
+	    Support for ksarchive.com has been added.  Thanks for Jade AislinSam implementing this.
 	  </p>
+	  <p>
+	    Questions?  Check out our new 
+	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.  Thanks to Wyndham for writing these.
+	    </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-3-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-3.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -126,6 +130,13 @@
 	  </p>
       </div>
       <div id='helpbox'>
+	<h3>Supported sites:</h3>
+	<p>
+	  There's a 
+ 	  <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">Supported
+ 	  Sites</a> page in our wiki.  If you have a site you'd like
+ 	  to see supported, please check there first.
+	</p>
 	<dl>
 	  <dt>fictionalley.org</dt>
 	  <dd>
@@ -239,9 +250,42 @@
 	  </dd>
 	  <dt>gayauthors.org</dt>
 	  <dd>
-	    Use the URL of the story, or one of it's chapters, such as
-	    <br /><a href="http://www.gayauthors.org/story/mark-arbour/stvincent">http://www.gayauthors.org/story/mark-arbour/stvincent</a>.
-	    <br /><a href="http://www.gayauthors.org/story/Mark%20Arbour/stvincent/7">http://www.gayauthors.org/story/Mark Arbour/stvincent/7</a>.
+	    Removed following complaints by the site administration.
+	  </dd>
+	  <dt>fanfiction.mugglenet.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.mugglenet.com/viewstory.php?sid=123">http://fanfiction.mugglenet.com/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>www.hpfandom.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.hpfandom.net/eff/viewstory.php?sid=123">http://www.hpfandom.net/eff/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>thequidditchpitch.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thequidditchpitch.org/viewstory.php?sid=123">http://thequidditchpitch.org/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>fanfiction.portkey.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.portkey.org/story/123">http://fanfiction.portkey.org/story/123</a>.
+	  </dd>
+	  <dt>nfacommunity.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://nfacommunity.com/viewstory.php?sid=1654">http://nfacommunity.com/viewstory.php?sid=1654</a>.
+	  </dd>
+	  <dt>www.midnightwhispers.ca</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.midnightwhispers.ca/viewstory.php?sid=1124">http://www.midnightwhispers.ca/viewstory.php?sid=1124</a>.
+	  </dd>
+	  <dt>ksarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://ksarchive.com/viewstory.php?sid=1124">http://ksarchive.com/viewstory.php?sid=1124</a>.
 	  </dd>
 	</dl>
 	<p>
@@ -253,6 +297,7 @@
 	    is being verified by Google and is absolutely, totally unknown to anyone but you.
 	  </li>
 	  <li>
+	    
 	    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by me</a>
 	    &mdash; how to read fiction in Stanza or any other ebook reader.
 	  </li>

plugin-defaults.ini

@@ -122,6 +122,11 @@ extratags: FanFiction
 # (.*)Great(.*)=>\1Moderate\2
 # .*-Centered=>
 
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -216,15 +221,28 @@ output_css:
 ## in chapters.
 #make_firstimage_cover: false
 
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
 ## If set, and there isn't already a cover image from the adapter or
 ## from make_firstimage_cover, this image will be made the cover.
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
 ## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
 #default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
 #default_cover_image:http://www.somesite.com/someimage.gif
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 ## Resize images down to width, height, preserving aspect ratio.
 ## Nook size, with margin.
 image_max_size: 580, 725
@@ -233,6 +251,13 @@ image_max_size: 580, 725
 ## space.
 #grayscale_images: false
 
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
@@ -253,8 +278,92 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML
 
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.mugglenet.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.archiveofourown.org]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fanfiction.net]
 
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.fictionpress.com]
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
@@ -267,6 +376,74 @@ extratags:
 #username:YourName
 #password:yourpassword
 
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.mediaminer.org]
+
+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+[www.potionsandsnitches.net]
+
+[www.siye.co.uk]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -291,68 +468,6 @@ collect_series: false
 
 [www.whofic.com]
 
-[www.mediaminer.org]
-
-[www.thewriterscoffeeshop.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
-collect_series: false
-
-[www.ficwad.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-[www.adastrafanfic.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fictionalley.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.harrypotterfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fimfiction.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.tthfanfic.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-## tth is a little unusual--it doesn't require user/pass, but the site
-## keeps track of which chapters you've read and won't send another
-## update until it thinks you're up to date.  This way, on download,
-## it thinks you're up to date.
-#username:YourName
-#password:yourpassword
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

plugin-example.ini

@@ -7,6 +7,28 @@
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
 ## collections.  This lets us turn it on and off by site without
@@ -55,6 +77,7 @@
 
 [www.fimfiction.net]
 #is_adult:true
+#fail_on_password: false
 
 [www.tthfanfic.org]
 #is_adult:true

readme.txt

@@ -10,5 +10,10 @@ python downloader.py http://www.fanfiction.net/s/5192986/1/A_Fox_in_Tokyo
 
 Do 'python downloader.py -h' for more options.
 
-This tool uses Python 2.5.2, but should work with newer versions of Python.
+This tool uses Python 2.7, but should work with newer versions of Python.
 
+For more information, see:
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs