Mild fixes and integration for new adapters, bump versions.

destinysgateway.com, ncisfiction.com,
stargateatlantis.org, thealphagate.com

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-12
+version: 4-4-16
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 24)
+    version             = (1, 5, 33)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -39,6 +39,7 @@ all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
 
 all_prefs.defaults['updatemeta'] = True
 all_prefs.defaults['updatecover'] = False
+all_prefs.defaults['updateepubcover'] = False
 all_prefs.defaults['keeptags'] = False
 all_prefs.defaults['urlsfromclip'] = True
 all_prefs.defaults['updatedefault'] = True
@@ -67,6 +68,7 @@ all_prefs.defaults['custom_cols'] = {}
 copylist = ['personal.ini',
             'updatemeta',
             'updatecover',
+            'updateepubcover',
             'keeptags',
             'urlsfromclip',
             'updatedefault',
@@ -176,6 +178,7 @@ class ConfigWidget(QWidget):
         prefs['collision'] = unicode(self.basic_tab.collision.currentText())
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
+        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -245,9 +248,11 @@ class BasicTab(QWidget):
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
-        
+
+        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
         horz = QHBoxLayout()
         label = QLabel('Default Output &Format:')
+        label.setToolTip(tooltip)
         horz.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
@@ -255,15 +260,16 @@ class BasicTab(QWidget):
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(tooltip)
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
 
+        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
         horz = QHBoxLayout()
         label = QLabel('Default If Story Already Exists?')
-        label.setToolTip("What to do if there's already an existing story with the same title and author.")
+        label.setToolTip(tooltip)
         horz.addWidget(label)
         self.collision = QComboBox(self)
         # add collision options
@@ -271,18 +277,23 @@ class BasicTab(QWidget):
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        self.collision.setToolTip(tooltip)
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update title, author, URL, tags, custom columns, etc for story in Calibre from web site.')
+        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off.")
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.updatecover = QCheckBox('Update Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
+        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.l.addWidget(self.updateepubcover)
+
+        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
+        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
@@ -514,7 +525,7 @@ class GenerateCoverTab(QWidget):
             if site == u"Default":
                 s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
             else:
-                s = "On Metadata update, run Generate Cover with this setting for site (%s)."%site
+                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
 
             label.setToolTip(s)
             horz.addWidget(label)
@@ -536,7 +547,7 @@ class GenerateCoverTab(QWidget):
         self.l.addWidget(self.gcnewonly)
 
         self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
-        self.allow_gc_from_ini.setToolTip("The INI parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site,\nbut it's much more complex.  generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             

calibre-plugin/dialogs.py

@@ -83,7 +83,7 @@ class AddNewDialog(SizePersistedDialog):
 
         self.l.addWidget(QLabel('Story URL(s), one per line:'))
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
         self.l.addWidget(self.url)
@@ -123,10 +123,18 @@ class AddNewDialog(SizePersistedDialog):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
+        horz = QHBoxLayout()
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
         self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
+        horz.addWidget(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        horz.addWidget(self.updateepubcover)
+        
+        self.l.addLayout(horz)
 
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -155,6 +163,7 @@ class AddNewDialog(SizePersistedDialog):
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
             }
 
     def get_urlstext(self):
@@ -168,6 +177,43 @@ class FakeLineEdit():
     def text(self):
         pass
     
+class CollectURLDialog(QDialog):
+    '''
+    Collect single url for get urls.
+    '''
+    def __init__(self, gui, title):
+        QDialog.__init__(self, gui)
+        self.gui = gui
+        self.status=False
+
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle(title)
+        self.l.addWidget(QLabel(title),0,0,1,2)
+        
+        self.l.addWidget(QLabel("URL:"),1,0)
+        self.url = QLineEdit(self)
+        self.l.addWidget(self.url,1,1)
+   
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,2,0)
+
+        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,2,1)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
 class UserPassDialog(QDialog):
     '''
     Need to collect User/Pass for some sites.
@@ -402,6 +448,11 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.updatemeta.setChecked(prefs['updatemeta'])
         options_layout.addWidget(self.updatemeta)
                 
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        options_layout.addWidget(self.updateepubcover)
+        
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
@@ -435,6 +486,7 @@ class UpdateExistingDialog(SizePersistedDialog):
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
             }
 
 def display_story_list(gui, header, prefs, icon, books,

calibre-plugin/ffdl_plugin.py

@@ -37,11 +37,12 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
 from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, 
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
     NotGoingToDownload )
 
@@ -192,6 +193,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                             shortcut_name='Get URLs from Selected Books',
                                                             triggered=self.get_list_urls)
 
+            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                                                            unique_name='Get Story URLs from Web Page',
+                                                            shortcut_name='Get Story URLs from Web Page',
+                                                            triggered=self.get_urls_from_page)
+
             self.menu.addSeparator()
             self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
                                                            image= 'config.png',
@@ -246,6 +252,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 (prefs['addtolists'] or prefs['addtoreadlists']) :
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
+    def get_urls_from_page(self):
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page")
+        d.exec_()
+        if not d.status:
+            return
+        print("URL:%s"%d.url.text())
+
+        url_list = get_urls_from_page("%s"%d.url.text())
+
+        if url_list:
+            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d.setWindowIcon(get_icon('bookmarks.png'))
+            d.exec_()
+        else:
+            info_dialog(self.gui, _('List of URLs'),
+                        _('No Valid URLs found on given page.'),
+                        show=True,
+                        show_copy_button=False)
+        
+            
     def get_list_urls(self):
         if len(self.gui.library_view.get_selected_ids()) > 0:
             book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
@@ -386,7 +412,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def get_metadata_for_book(self,book,
                               options={'fileform':'epub',
                                        'collision':ADDNEW,
-                                       'updatemeta':True}):
+                                       'updatemeta':True,
+                                       'updateepubcover':True}):
         '''
         Update passed in book dict with metadata from website and
         necessary data.  To be called from LoopProgressDialog
@@ -402,6 +429,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         fileform  = options['fileform']
         collision = options['collision']
         updatemeta= options['updatemeta']
+        updateepubcover= options['updateepubcover']
 
         if not book['good']:
             # book has already been flagged bad for whatever reason.
@@ -496,7 +524,7 @@ make_firstimage_cover:true
 
             # find dups
             mi = MetaInformation(story.getMetadata("title", removeallentities=True),
-                                 (story.getMetadata("author", removeallentities=True),)) # author is a list.
+                                 [story.getMetadata("author", removeallentities=True)]) # author is a list.
             identicalbooks = db.find_identical_books(mi)
             ## removed for being overkill.
             # for ib in identicalbooks:
@@ -593,7 +621,8 @@ make_firstimage_cover:true
     def start_download_list(self,book_list,
                             options={'fileform':'epub',
                                      'collision':ADDNEW,
-                                     'updatemeta':True}):
+                                     'updatemeta':True,
+                                     'updateepubcover':True}):
         '''
         Called by LoopProgressDialog to start story downloads BG processing.
         adapter_list is a list of tuples of (url,adapter)
@@ -641,7 +670,8 @@ make_firstimage_cover:true
     def _update_book(self,book,db=None,
                      options={'fileform':'epub',
                               'collision':ADDNEW,
-                              'updatemeta':True}):
+                              'updatemeta':True,
+                              'updateepubcover':True}):
         print("add/update %s %s"%(book['title'],book['url']))
         mi = self._make_mi_from_book(book)
                 
@@ -716,8 +746,8 @@ make_firstimage_cover:true
                                    good_list,
                                    partial(self._update_book, options=options, db=self.gui.current_db),
                                    partial(self._update_books_completed, options=options),
-                                   init_label="Updating calibre for stories...",
-                                   win_title="Update calibre for stories",
+                                   init_label="Updating calibre for FanFiction stories...",
+                                   win_title="Update calibre for FanFiction stories",
                                    status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
@@ -780,7 +810,7 @@ make_firstimage_cover:true
             if epubmi.cover_data[1] is not None:
                 db.set_cover(book_id, epubmi.cover_data[1])
 
-        # set author link if found.  All current adapters have authorUrl.
+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         if 'authorUrl' in book['all_metadata']:
             autid=db.get_author_id(book['author'])
             db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
@@ -902,10 +932,18 @@ make_firstimage_cover:true
             if len(lists) < 1 :
                 message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
+
+            # Quick demo of how an 'allow send' list might work.
+            # Issues: allow list per send list?  Naming convention?  "send(allow)"
+            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
+            # # intersection of book_ids & allow_list
+            # add_book_ids = list(set(book_ids) & set(allow_list))
+
             for l in lists:
                 if l in rl_plugin.get_list_names():
                     #print("good send l:(%s)"%l)
                     rl_plugin.add_books_to_list(l,
+                                                #add_book_ids,
                                                 book_ids,
                                                 display_warnings=False)
                 else:
@@ -914,7 +952,7 @@ make_firstimage_cover:true
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
     def _find_existing_book_id(self,db,book,matchurl=True):
-        mi = MetaInformation(book["title"],(book["author"],)) # author is a list.
+        mi = MetaInformation(book["title"],[book["author"]]) # author is a list.
         identicalbooks = db.find_identical_books(mi)
         if matchurl: # only *really* identical if URL matches, too.
             for ib in identicalbooks:
@@ -925,7 +963,7 @@ make_firstimage_cover:true
         return None
     
     def _make_mi_from_book(self,book):
-        mi = MetaInformation(book['title'],(book['author'],)) # author is a list.
+        mi = MetaInformation(book['title'],[book['author']]) # author is a list.
         mi.set_identifiers({'url':book['url']})
         mi.publisher = book['publisher']
         mi.tags = book['tags']
@@ -941,7 +979,15 @@ make_firstimage_cover:true
         books = []
         uniqueurls = set()
         for url in urls:
+            # look here for [\d,\d] at end of url, and remove?
+            mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+            print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+            url = mc.group('url')
             book = self._convert_url_to_book(url)
+            book['begin'] = mc.group('begin')
+            book['end'] = mc.group('end')
+            if book['begin'] and not mc.group('comma'):
+                book['end'] = book['begin']
             if book['url'] in uniqueurls:
                 book['good'] = False
                 book['comment'] = "Same story already included."
@@ -956,6 +1002,8 @@ make_firstimage_cover:true
         book['title'] = 'Unknown'
         book['author'] = 'Unknown'
         book['author_sort'] = 'Unknown'
+        book['begin'] = None
+        book['end'] = None
             
         book['comment'] = ''
         book['url'] = ''
@@ -971,6 +1019,8 @@ make_firstimage_cover:true
         book['title'] = 'Unknown'
         book['author'] = 'Unknown'
         book['author_sort'] = 'Unknown'
+        book['begin'] = None
+        book['end'] = None
             
         book['comment'] = ''
         book['url'] = ''

calibre-plugin/jobs.py

@@ -87,7 +87,15 @@ def do_download_worker(book_list, options,
         print(job.details)
 
         if count >= total:
-            # All done!
+            # All done!  Output some lists for convenience of some users.
+            print("Successfully downloaded:")
+            for book in book_list:
+                if book['good']:
+                    print(book['title'])
+            print("\nUnsuccessful:")
+            for book in book_list:
+                if not book['good']:
+                    print(book['title'])
             break
 
     server.close()
@@ -107,6 +115,9 @@ def do_download_for_worker(book,options):
         ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
         ffdlconfig.readfp(StringIO(options['personal.ini']))
         
+        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            ffdlconfig.set("overrides","never_make_cover","true")
+        
         adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
         adapter.is_adult = book['is_adult'] 
         adapter.username = book['username'] 
@@ -114,10 +125,10 @@ def do_download_for_worker(book,options):
         
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
-            print("calibre_series:%s [%d]"%book['calibre_series'])
+            # print("calibre_series:%s [%d]"%book['calibre_series'])
             adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
-        else:
-            print("no calibre_series")
+        # else:
+        #     print("no calibre_series")
         writer = writers.getWriter(options['fileform'],adapter.config,adapter)
 
         outfile = book['outfile']
@@ -130,19 +141,24 @@ def do_download_for_worker(book,options):
         ## checks were done earlier, it's new or not dup or newer--just write it.
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+
+            adapter.setChaptersRange(book['begin'],book['end'])
+            
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
         ## checks were done earlier, just update it.
         elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            
+
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
             urlchaptercount = int(story.getMetadata('numChapters'))
             (url,chaptercount,
              adapter.oldchapters,
-             adapter.oldimgs) = get_update_data(book['epub_for_update'])
+             adapter.oldimgs,
+             adapter.oldcover,
+             adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
 
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)

defaults.ini

@@ -183,6 +183,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -409,6 +412,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[onedirectionfanfiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[samdean.archive.nu]
+
+[stargate-atlantis.org]
+
 [thehexfiles.net]
 
 [thequidditchpitch.org]
@@ -430,6 +450,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.thealphagate.com]
+
 [www.archiveofourown.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -444,7 +466,18 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.destinysgateway.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.dokuga.com]
+
 [www.fanfiction.net]
+## fanfiction.net's 'cover' images are really just tiny thumbnails.
+## Comment this out or change it to false to use them anyway.
+never_make_cover: true
 
 [www.ficbook.net]
 
@@ -492,6 +525,19 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.ik-eternal.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.libraryofmoria.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -510,6 +556,8 @@ extratags:
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
+[www.ncisfiction.com]
+
 [www.phoenixsong.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -520,6 +568,8 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.potionsandsnitches.net]
 
+[www.prisonbreakfic.net]
+
 [www.siye.co.uk]
 
 [www.squidge.org/peja]
@@ -533,6 +583,8 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
+[www.storiesofarda.com]
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -598,6 +650,12 @@ collect_series: false
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.yourfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -30,6 +30,7 @@ from subprocess import call
 
 from fanficdownloader import adapters,writers,exceptions
 from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+from fanficdownloader.geturls import get_urls_from_page
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -64,9 +65,15 @@ def main():
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
                      help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+   parser.add_option("--update-cover",
+                     action="store_true", dest="updatecover",
+                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
                      help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+   parser.add_option("-l", "--list",
+                     action="store_true", dest="list",
+                     help="Get list of valid story URLs from page given.",)
    
    (options, args) = parser.parse_args()
 
@@ -104,12 +111,21 @@ def main():
 
    if options.force:
        config.set("overrides","always_overwrite","true")
+
+   if options.update and not options.updatecover:
+       config.set("overrides","never_make_cover","true")
        
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
            config.set("overrides",var,val)
 
+   if options.list:
+       retlist = get_urls_from_page(args[0])
+       print "\n".join(retlist)
+               
+       return
+
    try:
        ## Attempt to update an existing epub.
        if options.update:
@@ -167,7 +183,9 @@ def main():
                    # merging epubs.
                    (url,chaptercount,
                     adapter.oldchapters,
-                    adapter.oldimgs) = get_update_data(args[0])
+                    adapter.oldimgs,
+                    adapter.oldcover,
+                    adapter.calibrebookmark) = get_update_data(args[0])
 
                    writeStory(config,adapter,"epub")
                    
@@ -194,7 +212,7 @@ def main():
        print dne
    except exceptions.UnknownSite, us:
        print us
-   
+
 if __name__ == "__main__":
     #import time
     #start = time.time()

fanficdownloader/adapters/__init__.py

@@ -65,6 +65,17 @@ import adapter_phoenixsongnet
 import adapter_walkingtheplankorg
 import adapter_ashwindersycophanthexcom
 import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_prisonbreakficnet
+import adapter_storiesofardacom
+import adapter_yourfanfictioncom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictioncom
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -106,9 +106,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 		
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.text)
+        if a == None: # ao3 allows for author 'Anonymous' with no author link.
+            self.story.setMetadata('author','Anonymous')
+            self.story.setMetadata('authorUrl',self.url)
+        else:
+            self.story.setMetadata('authorId',a['href'].split('/')[2])
+            self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+            self.story.setMetadata('author',a.text)
 
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -104,7 +104,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
     
         d = self._fetchUrl(loginUrl, params)
     
-        if "User Account Page" not in d : #Member Account
+        if "Logout" not in d : #Member Account
             logging.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
@@ -157,10 +157,10 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         except:
             pass
 		
-        for info in asoup.findAll('table', {'bordercolor' : '#1A1919'}):
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',a.string)
                 break
 		
 
@@ -189,7 +189,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-			
         a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
         val = a.nextSibling
         svalue = ""

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DestinysGatewayComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DestinysGatewayComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dgrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.destinysgateway.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dokugacom.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DokugaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DokugaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dkg')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'fanfiction' in self.story.getMetadata('section'):
+            self.dateformat = "%d %b %Y"
+        else:
+            self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dokuga.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'align' : 'center'}).find('h3')
+        
+        # Find authorid and URL from... author url.
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        alink='http://'+self.host+aut['href']
+        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        a = a.string[:(len(a.string)-4)]
+        self.story.setMetadata('title',a)
+
+        # Find the chapters:
+        chapters = soup.find('select').findAll('option')
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
+        
+        if 'fanfiction' in self.story.getMetadata('section'):
+            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
+        
+            #grab the rest of the metadata from the author's page
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+            div=div.nextSibling
+            self.setDescription(url,div)
+        
+            div=div.nextSibling
+            
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split('&')[0])
+        
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split('&')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Category: ')
+            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
+            self.story.addToList('category', 'Fanfiction')
+        
+            a=div.text.split('Created: ')
+            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
+        
+            a=div.text.split('Updated: ')
+            if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1]), self.dateformat))
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Words: ')
+            if len(a) == 2: self.story.setMetadata('numWords', a[1].split('&')[0])
+        
+            a=div.text.split('Genre: ')
+            if len(a) == 2:
+                for genre in a[1].split('&')[0].split(', '):
+                    self.story.addToList('genre',genre)
+        
+        else:
+            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+                    
+            div=div.nextSibling.nextSibling
+            self.setDescription(url,div)
+            self.story.addToList('category', 'Spark')
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split(' - ')[0])
+            
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split(' - ')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Genre: ')
+            if len(a)==2:
+                for genre in a[1].split(' - ')[0].split('/'):
+                    self.story.addToList('genre',genre)
+                
+            div=div.nextSibling.nextSibling
+            
+            a=div.text.split('Updated: ')
+            if len(a)==2:
+                date=a[1].split('      -')[0]
+                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        
+                # does not have published date anywhere
+                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            
+            a=div.text.split('Words ')
+            if len(a)==2: self.story.setMetadata('numWords', a[1])
+            
+        
+
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chtext'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -23,9 +23,14 @@ import time
 
 from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
+from ..htmlcleanup import stripHTML
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
+ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
+             "Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
+             "Spiritual", "Supernatural", "Suspense", "Tragedy", "Western"]
+
 class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
@@ -198,55 +203,52 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## Pull some additional data from html.  Find Rating and look around it.
 
         a = soup.find('a', href='http://www.fictionratings.com/')
-        self.story.setMetadata('rating',a.string)
-
-        # used below to get correct characters.
-        metatext = a.findNext(text=re.compile(r' - Reviews:'))
-        if metatext == None: # indicates there's no Reviews, look for id: instead.
-            metatext = a.findNext(text=re.compile(r' - id:'))
+        rating = a.string
+        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
+            rating = rating[8:]
+            
+        self.story.setMetadata('rating',rating)
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
+        gui_table1i = soup.find(id="gui_table1i")
+        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+        metalist = metatext.split(" - ")
+        #print("metatext:(%s)"%metalist)
+
+        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+
+        # rating is obtained above more robustly.
+        if metalist[0].startswith('Rated:'):
+            metalist=metalist[1:]
+
+        # next is assumed to be language.
+        self.story.setMetadata('language',metalist[0])
+        metalist=metalist[1:]
+
+        # next might be genre.
+        genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
+        goodgenres=True
+        for g in genrelist:
+            if g not in ffnetgenres:
+                goodgenres=False
+        if goodgenres:
+            self.story.extendList('genre',genrelist)
+            metalist=metalist[1:]
+
+        # next might be characters, otherwise Reviews, Updated, Published, Words
+        if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') ):
+            self.story.extendList('characters',metalist[0].split('&'))
+        
+        if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
-        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
-        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
-        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
-                     soup.find('meta',{'name':'description'})['content'])
-        if m != None:
-            genres=m.group('genres')
-            if genres != None:
-                # Hurt/Comfort is one genre.
-                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-                for g in genres.split('/'):
-                    self.story.addToList('genre',g)
-
-            if m.group('chars') != None:
-
-                # At this point we've proven that there's character(s)
-                # We can't reliably parse characters out of meta name="description".
-                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
-                # But we can pull them from the reviewstext line, now that we know about existance of chars.
-                # reviewstext can take form of:
-                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
-                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
-                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
-                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
-                              metatext)
-                chars = mc.group("chars")
-                for c in chars.split(' & '):
-                    self.story.addToList('characters',c)
-        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
-        if m.group('lang') != None:
-            self.story.setMetadata('language',m.group('lang'))
-                    
+        img = soup.find('img',{'class':'cimage'})
+        if img:
+            self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
+            
         return
 
     def getChapterText(self, url):
@@ -262,13 +264,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
         if sharediv:
             sharediv.extract()
-        else:
-            logging.debug('share button div not found')
         
-        div = soup.find('div', {'id' : 'storytext'})
+        div = soup.find('div', {'id' : 'storytextp'})
         
         if None == div:
-            logging.debug('div id=storytext not found.  data:%s'%data)
+            logging.debug('div id=storytextp not found.  data:%s'%data)
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return IkEternalNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class IkEternalNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ike')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ik-eternal.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        asoup = soup.find('div', {'class': 'listbox'})
+        for a in asoup.findAll('p'):
+            a.name='br'
+        labels = asoup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisfictioncom.py

@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["iso-8859-1",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisfn')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ncisfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulStoneSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'class' : 'main_title'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        
+        aut.extract()
+        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
+
+        # Find the chapters:
+        i=0
+        chapters=soup.findAll('table', {'class' : 'story_table'})
+        for chapter in chapters:
+            ch=chapter.find('a')
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
+            if i == 0:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            if i == len(chapters)-1:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            i=i+1
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        info = soup.find('table', {'class' : 'story_info'})
+        
+        # no convenient way to calculate word count as it is logged differently for stories with and without series
+        
+        labels = info.findAll('tr')
+        for tr in labels:
+            value = tr.find('td')
+            label = tr.find('th').string
+
+            if 'Summary' in label:
+                self.setDescription(url,value)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Category' in label:
+                cats = value.findAll('a')
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = value.findAll('a')
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = value.findAll('a')
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = value.findAll('a')
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'not completed' in value.text:
+                    self.story.setMetadata('status', 'In-Progress')
+                else:
+                    self.story.setMetadata('status', 'Completed')
+                    
+        try:
+            # Find Series name from series URL.
+            a = soup.find('div',{'class' : 'sub_header'})
+            series_name = a.find('a').string
+            i = a.text.split('#')[1]
+            self.setSeries(series_name, i)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'story_text'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return OneDirectionFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','odf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","One Direction")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'onedirectionfanfiction.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
+
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+        
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_prisonbreakficnet.py

@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PrisonBreakFicNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pbf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Prison Break")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.prisonbreakfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SamDeanArchiveNuAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SamDeanArchiveNuAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sda')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Supernatural")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'samdean.archive.nu'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.samdean.archive.nu','samdean.archive.nu']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. http://samdean.archive.nu/viewstory.php?sid=4317&chapter=2
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -174,7 +174,16 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
-        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
+                
+        # SIYE formats stories in the author list differently when their part of a series.
+        # Look for non-series...
+        divdesc = titlea.parent.parent.find('div',{'class':'desc'})
+        if not divdesc:
+            # ... now look for series.
+            divdesc = titlea.parent.parent.findNextSibling('tr').find('div',{'class':'desc'})
+
+        moremeta = stripHTML(divdesc)
+        #print("moremeta:%s"%moremeta)
         for part in moremeta.replace(' - ','\n').split('\n'):
             #print("part:%s"%part)
             try:

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -84,7 +84,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StargateAtlantisOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StargateAtlantisOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','stat')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Stargate: Atlantis")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'stargate-atlantis.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfics/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.findAll('div', {'class' : 'content'})
+        if len(chapters) > 1:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. 
+                link = chapter.find('a')
+                self.chapterUrls.append((stripHTML(link),'http://'+self.host+'/fanfics/'+link['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'tail' and value != None:
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOfArdaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOfArdaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','soa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Lord of the Rings")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.storiesofarda.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('th', {'colspan' : '3'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        a.find('em').extract()
+        self.story.setMetadata('title',a.text)
+
+        # Find the chapters: chapterview.asp?sid=7000&cid=30919
+        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        summary = soup.find('td', {'colspan' : '3'})
+        self.setDescription(url,summary)
+        
+        # no convenient way to get word count
+
+        for td in asoup.findAll('td', {'colspan' : '3'}):
+            if td.find('a', href=re.compile('chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None:
+                break     
+        td=td.nextSibling.nextSibling
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat))
+        tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling
+        td=tr.findAll('td')
+        self.story.setMetadata('rating', td[0].string.split(': ')[1])
+        self.story.setMetadata('status', td[2].string.split(': ')[1])
+        self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'width' : '90%'}).find('td')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheAlphaGateComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheAlphaGateComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tag')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Stargate: SG-1")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thealphagate.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -69,13 +69,15 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'thehexfiles.net'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thehexfiles.net','thehexfiles.net']
+
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -196,4 +198,4 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return self.utf8FromSoup(url,soup)
+        return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_yourfanfictioncom.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return YourFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class YourFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','yff')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.yourfanfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                    
+            if 'Tags' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/base_adapter.py

@@ -87,6 +87,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterLast = None
         self.oldchapters = None
         self.oldimgs = None
+        self.oldcover = None # (data of existing cover html, data of existing cover image)
+        self.calibrebookmark = None
         ## order of preference for decoding.
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of
@@ -218,6 +220,15 @@ class BaseSiteAdapter(Configurable):
                                                                self.getConfig('allow_unsafe_filename')),
                                      self._fetchUrlRaw,
                                      cover=True)
+
+            # no new cover, set old cover, if there is one.
+            if not self.story.cover and self.oldcover:
+                self.story.oldcover = self.oldcover
+                
+            # cheesy way to carry calibre bookmark file forward across update.
+            if self.calibrebookmark:
+                self.story.calibrebookmark = self.calibrebookmark
+                
         return self.story
 
     def getStoryMetadataOnly(self):

fanficdownloader/epubutils.py

@@ -40,6 +40,50 @@ def get_update_data(inputio,
     ## Save the path to the .opf file--hrefs inside it are relative to it.
     relpath = get_path_part(rootfilename)
             
+    oldcover = None
+    calibrebookmark = None
+    # Looking for pre-existing cover.
+    for item in contentdom.getElementsByTagName("reference"):
+        if item.getAttribute("type") == "cover":
+            # there is a cover (x)html file, save the soup for it.
+            href=relpath+item.getAttribute("href")
+            oldcoverhtmlhref = href
+            oldcoverhtmldata = epub.read(href)
+            oldcoverhtmltype = "application/xhtml+xml"
+            for item in contentdom.getElementsByTagName("item"):
+                if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
+                    oldcoverhtmltype = item.getAttribute("media-type")
+                    break
+            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"))
+            src = None
+            # first img or image tag.
+            imgs = soup.findAll('img')
+            if imgs:
+                src = get_path_part(href)+imgs[0]['src']
+            else:
+                imgs = soup.findAll('image')
+                if imgs:
+                    src=get_path_part(href)+imgs[0]['xlink:href']
+
+            if not src:
+                continue
+            try:
+                # remove all .. and the path part above it, if present.
+                # Mostly for epubs edited by Sigil.
+                src = re.sub(r"([^/]+/\.\./)","",src)
+                print("epubutils: found pre-existing cover image:%s"%src)
+                oldcoverimghref = src
+                oldcoverimgdata = epub.read(src)
+                for item in contentdom.getElementsByTagName("item"):
+                    if( relpath+item.getAttribute("href") == oldcoverimghref ):
+                        oldcoverimgtype = item.getAttribute("media-type")
+                        break
+                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+            except Exception as e:
+                print("Cover Image %s not found"%src)
+                print("Exception: %s"%(unicode(e)))
+                traceback.print_exc()
+
     filecount = 0
     soups = [] # list of xhmtl blocks
     images = {} # dict() longdesc->data
@@ -61,7 +105,7 @@ def get_update_data(inputio,
                             try:
                                 newsrc=get_path_part(href)+img['src']
                                 # remove all .. and the path part above it, if present.
-                                # Most for epubs edited by Sigil.
+                                # Mostly for epubs edited by Sigil.
                                 newsrc = re.sub(r"([^/]+/\.\./)","",newsrc)
                                 longdesc=img['longdesc']
                                 data = epub.read(newsrc)
@@ -85,9 +129,14 @@ def get_update_data(inputio,
                         
                     filecount+=1
 
+    try:
+        calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt")
+    except:
+        pass
+                    
     for k in images.keys():
         print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images)
+    return (source,filecount,soups,images,oldcover,calibrebookmark)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)

fanficdownloader/geturls.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import urlparse
+import urllib2 as u2
+import ConfigParser
+
+from BeautifulSoup import BeautifulSoup 
+from gziphttp import GZipProcessor
+
+import adapters
+
+def get_urls_from_page(url):
+    
+    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+    soup = BeautifulSoup(opener.open(url).read())
+    
+    normalized = set() # normalized url
+    retlist = [] # orig urls.
+    config = ConfigParser.SafeConfigParser()
+    
+    for a in soup.findAll('a'):
+        if a.has_key('href'):
+            href = form_url(url,a['href'])
+            try:
+                adapter = adapters.getAdapter(config,href,"EPUB")
+                if adapter.story.getMetadata('storyUrl') not in normalized:
+                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    retlist.append(href)
+            except:
+                pass
+
+    return retlist
+
+def form_url(parenturl,url):
+     url = url.strip() # ran across an image with a space in the
+                       # src. Browser handled it, so we'd better, too.
+ 
+     if "//" in url or parenturl == None:
+         returl = url
+     else:
+         parsedUrl = urlparse.urlparse(parenturl)
+         if url.startswith("/") :
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  url,
+                  '','',''))
+         else:
+             toppath=""
+             if parsedUrl.path.endswith("/"):
+                 toppath = parsedUrl.path
+             else:
+                 toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  toppath + '/' + url,
+                  '','',''))
+     return returl
+       

fanficdownloader/html2text.py

@@ -103,9 +103,10 @@ def onlywhite(line):
             return c is ' '
     return line
 
-def optwrap(text):
+def optwrap(text,wrap_width=BODY_WIDTH):
     """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
+    
+    if not wrap_width:
         return text
     
     assert wrap, "Requires Python 2.3."
@@ -114,7 +115,7 @@ def optwrap(text):
     for para in text.split("\n"):
         if len(para) > 0:
             if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
+                for line in wrap(para, wrap_width):
                     result += line + "\n"
                 result += "\n"
                 newlines = 2
@@ -423,8 +424,8 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
     h.feed("")
     return h.close()
 
-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
+def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
+    return optwrap(html2text_file(html, None, baseurl),wrap_width)
 
 if __name__ == "__main__":
     baseurl = ''

fanficdownloader/story.py

@@ -197,7 +197,9 @@ class Story:
         self.imgurls = []
         self.imgtuples = []
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
-        self.cover=None
+        self.cover=None # *href* of new cover image--need to create html.
+        self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+        self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
@@ -250,7 +252,12 @@ class Story:
             allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
 
         return allmetadata
-        
+
+    # just for less clutter in adapters.
+    def extendList(self,listname,l):
+        for v in l:
+            self.addToList(listname,v)
+    
     def addToList(self,listname,value):
         if value==None:
             return

fanficdownloader/writers/writer_epub.py

@@ -262,6 +262,37 @@ ${value}<br />
         items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                    ## but it needs to be in the items manifest.
 
+        guide = None
+        coverIO = None
+                
+        imgid = "image0000"
+        if not self.story.cover and self.story.oldcover:
+            print("writer_epub: no new cover, has old cover, write image.")
+            (oldcoverhtmlhref,
+             oldcoverhtmltype,
+             oldcoverhtmldata,
+             oldcoverimghref,
+             oldcoverimgtype,
+             oldcoverimgdata) = self.story.oldcover
+            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
+            outputepub.writestr(oldcoverimghref,oldcoverimgdata)
+            
+            imgid = "image0"
+            items.append((imgid,
+                          oldcoverimghref,
+                          oldcoverimgtype,
+                          None))
+            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
+            itemrefs.append("cover")
+            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
+                                                           "name":"cover"}))
+            guide = newTag(contentdom,"guide")
+            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
+                                                                   "title":"Cover",
+                                                                   "href":oldcoverhtmlhref}))
+            
+            
+
         if self.getConfig('include_images'):
             imgcount=0
             for imgmap in self.story.getImgUrls():
@@ -276,9 +307,6 @@ ${value}<br />
         
         items.append(("style","OEBPS/stylesheet.css","text/css",None))
 
-        guide = None
-        coverIO = None
-
         if self.story.cover:
             # Note that the id of the cover xhmtl *must* be 'cover'
             # for it to work on Nook.
@@ -347,8 +375,8 @@ div { margin: 0pt; padding: 0pt; }
         contentxml = contentdom.toxml(encoding='utf-8')
         
         # tweak for brain damaged Nook STR.  Nook insists on name before content.
-        contentxml = contentxml.replace('<meta content="image0000" name="cover"/>',
-                                        '<meta name="cover" content="image0000"/>')
+        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%imgid,
+                                        '<meta name="cover" content="%s"/>'%imgid)
         outputepub.writestr("content.opf",contentxml)
 
         contentdom.unlink()
@@ -458,6 +486,9 @@ div { margin: 0pt; padding: 0pt; }
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml
 
+        if self.story.calibrebookmark:
+            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)
+
 	# declares all the files created by Windows.  otherwise, when
         # it runs in appengine, windows unzips the files as 000 perms.
         for zf in outputepub.filelist:

fanficdownloader/writers/writer_txt.py

@@ -21,7 +21,7 @@ from textwrap import wrap
 
 from base_writer import *
 
-from ..html2text import html2text, BODY_WIDTH
+from ..html2text import html2text
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the
@@ -106,6 +106,12 @@ End file.
 
     def writeStoryImpl(self, out):
 
+        self.wrap_width = self.getConfig('wrap_width')
+        if self.wrap_width == '' or self.wrap_width == '0':
+            self.wrap_width = None
+        else:
+            self.wrap_width = int(self.wrap_width)
+        
         wrapout = KludgeStringIO()
         
         wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
@@ -131,15 +137,19 @@ End file.
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
-                self._write(out,self.lineends(html2text(html)))
+                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
 
         self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
 
     def wraplines(self, text):
+        
+        if not self.wrap_width:
+            return text
+        
         result=''
         for para in text.split("\n"):
             first=True
-            for line in wrap(para, BODY_WIDTH):
+            for line in wrap(para, self.wrap_width):
                 if first:
                     first=False
                 else:

index.html

@@ -54,9 +54,14 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Site</h3>
+	  <h3>New Sites</h3>
 	  <p>
-	    Now supporting thehexfiles.net, that brings us up to an even 40 supported sites.  Thanks again, Ida.
+	    
+	    New sites samdean.archive.nu, www.yourfanfiction.com,
+	    www.destinysgateway.com, www.thealphagate.com,
+	    stargate-atlantis.org and www.ncisfiction.com added.
+	    That's 51 different supported sites now.  Thanks, Ida!
+	    
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -66,7 +71,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-11.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-15.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -358,6 +363,64 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://thehexfiles.net/viewstory.php?sid=1234">http://thehexfiles.net/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>www.dokuga.com</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.dokuga.com/fanfiction/story/1234/1">http://www.dokuga.com/fanfiction/story/1234/1</a> or
+	    <br /><a href="http://www.dokuga.com/spark/story/1234/1">http://www.dokuga.com/spark/story/1234/1</a>
+	  </dd>
+	  <dt>www.ik-eternal.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.ik-eternal.net/viewstory.php?sid=1234">http://www.ik-eternal.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>onedirectionfanfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://onedirectionfanfiction.com/viewstory.php?sid=1234">http://onedirectionfanfiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.prisonbreakfic.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.prisonbreakfic.net/viewstory.php?sid=1234">http://www.prisonbreakfic.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.storiesofarda.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.storiesofarda.com/chapterlistview.asp?SID=7000">http://www.storiesofarda.com/chapterlistview.asp?SID=7000</a>
+	  </dd>
+	  <dt>samdean.archive.nu</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://samdean.archive.nu/viewstory.php?sid=1234">http://samdean.archive.nu/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.yourfanfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.yourfanfiction.com/viewstory.php?sid=1234">http://www.yourfanfiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.destinysgateway.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.destinysgateway.com/viewstory.php?sid=1234">http://www.destinysgateway.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.thealphagate.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.thealphagate.com/viewstory.php?sid=1234">http://www.thealphagate.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>stargate-atlantis.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://stargate-atlantis.org/fanfics/viewstory.php?sid=1234">http://stargate-atlantis.org/fanfics/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.ncisfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.ncisfiction.com/story.php?stid=1234">http://www.ncisfiction.com/story.php?stid=1234</a>
+	    Or use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.ncisfiction.com/chapters.php?stid=1234">http://www.ncisfiction.com/chapters.php?stid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -172,6 +172,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
 
@@ -395,6 +398,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[onedirectionfanfiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[samdean.archive.nu]
+
+[stargate-atlantis.org]
+
 [thehexfiles.net]
 
 [thequidditchpitch.org]
@@ -416,6 +436,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.thealphagate.com]
+
 [www.archiveofourown.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -430,7 +452,18 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[www.destinysgateway.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.dokuga.com]
+
 [www.fanfiction.net]
+## fanfiction.net's 'cover' images are really just tiny thumbnails.
+## Comment this out or change it to false to use them anyway.
+never_make_cover: true
 
 [www.ficbook.net]
 
@@ -475,6 +508,19 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.ik-eternal.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.libraryofmoria.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -493,6 +539,8 @@ extratags:
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
+[www.ncisfiction.com]
+
 [www.phoenixsong.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -503,6 +551,8 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.potionsandsnitches.net]
 
+[www.prisonbreakfic.net]
+
 [www.siye.co.uk]
 
 [www.squidge.org/peja]
@@ -516,6 +566,8 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
+[www.storiesofarda.com]
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -581,6 +633,12 @@ collect_series: false
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.yourfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For