Compare commits

...
21 Commits
Author SHA1 Message Date
Jim Miller cae4a74578 Bump version to v2.2.10 2015-07-14 13:33:05 -05:00
Jim Miller 3839bba182 Update translations. 2015-07-14 13:30:31 -05:00
Jim Miller 4338f4e550 Fix 'kludgey text find for older mediaminer story'. 2015-07-10 16:56:39 -05:00
Jim Miller 883f9e22cc Yet more tweaks to mediaminer.org URL detection. 2015-07-10 16:44:10 -05:00
Jim Miller 9b468da598 Make mediaminer.org URLs more flexible. 2015-07-10 12:19:27 -05:00
Jim Miller fa90a3f23c Updates for mediaminer.org changes. 2015-07-08 12:04:29 -05:00
Jim Miller 4f7fd93b64 Make searching for existing books by title/author(s) optional. 2015-07-07 23:18:46 -05:00
Jim Miller d37dadf972 Update translations. 2015-06-30 22:49:22 -05:00
Jim Miller 5dbbc2efe5 Strip imap config options of lead/trail spaces. 2015-06-29 10:29:28 -05:00
Jim Miller c442feeb26 Add feature - make it optional to set the Calibre Author URL (on Standard Columns tab) 2015-06-26 12:47:01 -05:00
Jim Miller 17b0800242 Add feature - make it optional to set the Calibre Author URL (on Standard Columns tab) 2015-06-26 12:46:14 -05:00
Jim Miller 3c6a60f001 Bump version to v2.2.9 2015-06-25 13:39:02 -05:00
Jim Miller 96529571b2 Update translations. 2015-06-25 13:22:06 -05:00
Jim Miller ab2eb447e2 Exclude doReplacements on add_genre_when_multi_category call to getList('category'). 2015-06-22 09:46:33 -05:00
Jim Miller 71a44e4e64 Default bulk_load true for all adapters. 2015-06-20 12:50:18 -05:00
Jim Miller ff42cd86e2 Update translations, bulk_load for NHA, Rated==Rating in eFiction base. 2015-06-20 12:48:16 -05:00
Jim Miller e2c34eaea1 Update adapter_nhamagicalworldsus, make a Base eFiction adapter. 2015-06-16 07:13:12 -05:00
Jim Miller 78e5d8427b Make passing Calibre Columns in optional and only pass savemetacol data when column is configured to reduce data passed to/from BG processes. 2015-06-12 19:39:30 -05:00
Jim Miller c0adf8e027 Bump version to v2.2.8 2015-06-09 17:56:11 -05:00
Jim Miller 69d1ce6c01 Preserve order of URLs fetched from page--especially important for anthologies. 2015-06-09 13:08:56 -05:00
Jim Miller 390c661a88 Preserve order of URLs fetched from page--especially important for anthologies. 2015-06-09 13:00:43 -05:00
22 changed files with 2707 additions and 2687 deletions
+1 -1
View File
@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
description = _('UI plugin to download FanFiction stories from various sites.')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Jim Miller'
version = (2, 2, 7)
version = (2, 2, 10)
minimum_calibre_version = (1, 48, 0)
#: This field defines the GUI plugin class that contains all the code
+31 -6
View File
@@ -259,6 +259,7 @@ class ConfigWidget(QWidget):
prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
@@ -280,6 +281,8 @@ class ConfigWidget(QWidget):
# if they've removed everything, reset to default.
prefs['personal.ini'] = get_resources('plugin-example.ini')
prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
# Covers tab
prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
# for backward compatibility:
@@ -321,6 +324,8 @@ class ConfigWidget(QWidget):
colsnewonly[col] = checkbox.isChecked()
prefs['std_cols_newonly'] = colsnewonly
prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
# Custom Columns tab
# error column
prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
@@ -344,10 +349,10 @@ class ConfigWidget(QWidget):
prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
prefs['imappass'] = unicode(self.imap_tab.imappass.text())
prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
@@ -517,6 +522,11 @@ class BasicTab(QWidget):
self.injectseries.setChecked(prefs['injectseries'])
self.l.addWidget(self.injectseries)
self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier. But if not found, it can also search existing books by Title and Author(s)."))
self.matchtitleauth.setChecked(prefs['matchtitleauth'])
self.l.addWidget(self.matchtitleauth)
rej_gb = groupbox = QGroupBox(_("Reject List"))
self.l = QVBoxLayout()
groupbox.setLayout(self.l)
@@ -655,8 +665,13 @@ class PersonalIniTab(QWidget):
self.defaults.clicked.connect(self.show_defaults)
self.l.addWidget(self.defaults)
self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
self.l.addWidget(self.cal_cols_pass_in)
self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
self.showcalcols.setToolTip(_("FanFicFare passes the Calibre columns into the download/update process. This will show you the columns available by name."))
self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
self.showcalcols.clicked.connect(self.show_showcalcols)
self.l.addWidget(self.showcalcols)
@@ -1223,7 +1238,6 @@ class CustomColumnsTab(QWidget):
self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
self.l.addWidget(self.allow_custcol_from_ini)
self.l.addSpacing(5)
label = QLabel(_("Special column:"))
label.setWordWrap(True)
self.l.addWidget(label)
@@ -1305,6 +1319,17 @@ class StandardColumnsTab(QWidget):
horz.addWidget(newonlycheck)
self.l.addLayout(horz)
self.l.addSpacing(5)
label = QLabel(_("Other Standard Column Options"))
label.setWordWrap(True)
self.l.addWidget(label)
self.l.addSpacing(5)
self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
self.set_author_url.setChecked(prefs['set_author_url'])
self.l.addWidget(self.set_author_url)
self.l.insertStretch(-1)
+50 -47
View File
@@ -828,6 +828,7 @@ class FanFicFarePlugin(InterfaceAction):
options['version'] = self.version
logger.debug(self.version)
options['personal.ini'] = get_fff_personalini()
options['savemetacol'] = prefs['savemetacol']
#print("prep_downloads:%s"%books)
@@ -1023,8 +1024,9 @@ class FanFicFarePlugin(InterfaceAction):
# all_metadata duplicates some data, but also includes extra_entries, etc.
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
# get metadata to save in configured column.
book['savemetacol'] = story.dump_html_metadata()
if prefs['savemetacol'] != '':
# get metadata to save in configured column.
book['savemetacol'] = story.dump_html_metadata()
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
@@ -1072,7 +1074,7 @@ class FanFicFarePlugin(InterfaceAction):
# try to find by identifier url or uri first.
identicalbooks = self.do_id_search(url)
# print("identicalbooks:%s"%identicalbooks)
if len(identicalbooks) < 1:
if len(identicalbooks) < 1 and prefs['matchtitleauth']:
# find dups
authlist = story.getList("author", removeallentities=True)
mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1202,51 +1204,52 @@ class FanFicFarePlugin(InterfaceAction):
#print("calibre_series:%s [%s]"%book['calibre_series'])
if book['good']: # there shouldn't be any !'good' books at this point.
## Filling calibre_std_* and calibre_cust_* metadata
book['calibre_columns']={}
# std columns
mi = db.get_metadata(book['calibre_id'],index_is_id=True)
# book['calibre_columns']['calibre_std_identifiers']=\
# {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
# 'label':_('Ids')}
for k in mi.standard_field_keys():
# for k in mi:
if k in STD_COLS_SKIP:
continue
(label,value,v,fmd) = mi.format_field_extended(k)
if not label and k in field_metadata:
label=field_metadata[k]['name']
key='calibre_std_'+k
# if k == 'user_categories':
# value=u', '.join(mi.get(k))
# label=_('User Categories')
if label: # only if it has a human readable name.
if value is None or not book['calibre_id']:
## if existing book, populate existing calibre column
## values in metadata, else '' to hide.
value=''
book['calibre_columns'][key]={'val':value,'label':label}
#logger.debug("%s(%s): %s"%(label,key,value))
# custom columns
for k, column in self.gui.library_view.model().custom_columns.iteritems():
if k != prefs['savemetacol']:
key='calibre_cust_'+k[1:]
label=column['name']
value=db.get_custom(book['calibre_id'],
label=column['label'],
index_is_id=True)
# custom always have name.
if value is None or not book['calibre_id']:
## if existing book, populate existing calibre column
## values in metadata, else '' to hide.
value=''
book['calibre_columns'][key]={'val':value,'label':label}
# logger.debug("%s(%s): %s"%(label,key,value))
if prefs['cal_cols_pass_in']:
# std columns
mi = db.get_metadata(book['calibre_id'],index_is_id=True)
# book['calibre_columns']['calibre_std_identifiers']=\
# {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
# 'label':_('Ids')}
for k in mi.standard_field_keys():
# for k in mi:
if k in STD_COLS_SKIP:
continue
(label,value,v,fmd) = mi.format_field_extended(k)
if not label and k in field_metadata:
label=field_metadata[k]['name']
key='calibre_std_'+k
# if k == 'user_categories':
# value=u', '.join(mi.get(k))
# label=_('User Categories')
if label: # only if it has a human readable name.
if value is None or not book['calibre_id']:
## if existing book, populate existing calibre column
## values in metadata, else '' to hide.
value=''
book['calibre_columns'][key]={'val':value,'label':label}
#logger.debug("%s(%s): %s"%(label,key,value))
# custom columns
for k, column in self.gui.library_view.model().custom_columns.iteritems():
if k != prefs['savemetacol']:
key='calibre_cust_'+k[1:]
label=column['name']
value=db.get_custom(book['calibre_id'],
label=column['label'],
index_is_id=True)
# custom always have name.
if value is None or not book['calibre_id']:
## if existing book, populate existing calibre column
## values in metadata, else '' to hide.
value=''
book['calibre_columns'][key]={'val':value,'label':label}
# logger.debug("%s(%s): %s"%(label,key,value))
# if still 'good', make a temp file to write the output to.
# For HTML format users, make the filename inside the zip something reasonable.
# For crazy long titles/authors, limit it to 200chars.
@@ -1738,7 +1741,7 @@ class FanFicFarePlugin(InterfaceAction):
# save metadata to configured column
if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
label = custom_columns[prefs['savemetacol']]['label']
self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata()
self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])
for col, meta in prefs['custom_cols'].iteritems():
@@ -1840,7 +1843,7 @@ class FanFicFarePlugin(InterfaceAction):
# set author link if found. All current adapters have authorUrl, except anonymous on AO3.
# Moved down so author's already in the DB.
if 'authorUrl' in book['all_metadata']:
if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
authurls = book['all_metadata']['authorUrl'].split(", ")
authorlist = [ a.replace('&',';') for a in book['author'] ]
authorids = db.new_api.get_item_ids('authors',authorlist)
+9 -5
View File
@@ -156,7 +156,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
book['comment'] = 'Metadata collected.'
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -176,7 +177,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -197,7 +199,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
if chaptercount == urlchaptercount:
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
return book
@@ -218,8 +221,9 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
+5 -40
View File
@@ -330,6 +330,11 @@ sort_ships:false
## User-agent
user_agent:FFF/2.X
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
## Each output format has a section that overrides [defaults]
[html]
@@ -766,11 +771,6 @@ extraships:Spike/Buffy
## this should go in your personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[dramione.org]
## Some sites require login (or login for some rated stories) The
## program can prompt you, or you can save it in config. In
@@ -951,11 +951,6 @@ extraships:Harry Potter/Hermione Granger
#username:YourName
#password:yourpassword
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings,romance
extra_titlepage_entries: readings,romance
readings_label: Readings
@@ -1103,11 +1098,6 @@ extracategories:Glee RPF
extracharacters:Darren Criss, Chris Colfer
extraships:Darren Criss/Chris Colfer
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[ksarchive.com]
## Site dedicated to these categories/characters/ships
extracategories:Star Trek
@@ -1127,11 +1117,6 @@ eroticatags_label:Erotica Tags
extra_titlepage_entries: eroticatags
[lotrfanfiction.com]
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings
readings_label: Readings
@@ -1381,11 +1366,6 @@ extracategories:Transgender
## confirm they are adult for adult content.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[thehexfiles.net]
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
@@ -1405,11 +1385,6 @@ extraships:Harry Potter/Draco Malfoy
## personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
## Site dedicated to these categories/characters/ships
extracategories:Criminal Minds
@@ -1419,11 +1394,6 @@ extracategories:Criminal Minds
## personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings,challenge
extra_titlepage_entries: readings,challenge
challenge_label: Challenge
@@ -1777,11 +1747,6 @@ extraships:InuYasha/Kagome
## Site dedicated to these categories/characters/ships
extracategories:Lord of the Rings
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[www.mediaminer.org]
[www.midnightwhispers.ca]
+3
View File
@@ -51,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
# take from here.
default_prefs = {}
default_prefs['personal.ini'] = get_resources('plugin-example.ini')
default_prefs['cal_cols_pass_in'] = False
default_prefs['rejecturls'] = ''
default_prefs['rejectreasons'] = '''Sucked
Boring
@@ -75,6 +76,7 @@ default_prefs['lookforurlinhtml'] = False
default_prefs['checkforseriesurlid'] = True
default_prefs['checkforurlchange'] = True
default_prefs['injectseries'] = False
default_prefs['matchtitleauth'] = True
default_prefs['smarten_punctuation'] = False
default_prefs['show_est_time'] = False
@@ -104,6 +106,7 @@ default_prefs['custom_cols_newonly'] = {}
default_prefs['allow_custcol_from_ini'] = True
default_prefs['std_cols_newonly'] = {}
default_prefs['set_author_url'] = True
default_prefs['imapserver'] = ''
default_prefs['imapuser'] = ''
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+34 -26
View File
@@ -22,7 +22,6 @@ import re
import urllib
import urllib2
from .. import BeautifulSoup as bs
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
# get storyId from url--url validation guarantees query correct
m = re.match(self.getSiteURLPattern(),url)
if m:
self.story.setMetadata('storyId',m.group('id'))
if m.group('id'):
self.story.setMetadata('storyId',m.group('id'))
elif m.group('id2'):
self.story.setMetadata('storyId',m.group('id2'))
elif m.group('id3'):
self.story.setMetadata('storyId',m.group('id2'))
# normalized story URL.
self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
def getSiteURLPattern(self):
## http://www.mediaminer.org/fanfic/view_st.php/76882
## http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
## http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
## http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
return re.escape("http://"+self.getSiteDomain())+\
"/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
r"/fanfic/view_(st|ch)\.php"+\
r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
# Override stripURLParameters so the id parameter won't get stripped
@classmethod
def stripURLParameters(cls, url):
return url
def extractChapterUrlsAndMetadata(self):
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
logger.debug("URL: "+url)
try:
data = self._fetchUrl(url)
data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
raise e
# use BeautifulSoup HTML parser to make everything easier to find.
soup = bs.BeautifulSoup(data)
soup = self.make_soup(data)
# [ A - All Readers ], strip '[' ']'
## Above title because we remove the smtxt font to get title.
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
title = soup.find('td',{'class':'ffh'})
for font in title.findAll('font'):
font.extract() # removes 'font' tags from inside the td.
if title.has_key('colspan'):
if title.has_attr('colspan'):
titlet = stripHTML(title)
else:
## No colspan, it's part chapter title--even if it's a one-shot.
titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
self.story.setMetadata('title',titlet)
## The story title is difficult to reliably parse from the
## story pages. Getting it from the author page is, but costs
## another fetch.
# authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
# titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
# self.story.setMetadata('title',titlea.text)
# save date from first for later.
firstdate=None
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
# save date from first for later.
if not firstdate:
firstdate = m.group(3)
self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
# http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
# self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
self.story.setMetadata('numChapters',len(self.chapterUrls))
# category
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
logger.debug('Getting chapter text from: %s' % url)
data=self._fetchUrl(url)
soup = bs.BeautifulStoneSoup(data,
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
soup = self.make_soup(data)
anchor = soup.find('a',{'name':'fic_c'})
header = soup.find('div',{'class':'post-meta clearfix '})
# print("data:%s"%data)
if None == anchor:
chapter=self.make_soup('<div class="story"></div>').find('div')
if None == header:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
## find divs with align=left, those are paragraphs in newer stories.
divlist = anchor.findAllNext('div',{'align':'left'})
divlist = header.findAllNext('div',{'align':'left'})
if divlist:
for div in divlist:
div.name='p' # convert to <p> mediaminer uses div with
# a margin for paragraphs.
anchor.append(div) # cheat! stuff all the content
# divs into anchor just as a
# holder.
chapter.append(div)
del div['style']
del div['align']
anchor.name='div'
return self.utf8FromSoup(url,anchor)
return self.utf8FromSoup(url,chapter)
else:
logger.debug('Using kludgey text find for older mediaminer story.')
## Some older mediaminer stories are unparsable with BeautifulSoup.
## Really nasty formatting. Sooo... Cheat! Parse it ourselves a bit first.
## Story stuff falls between:
data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
soup = bs.BeautifulStoneSoup(data,
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
soup = self.make_soup(data)
for tag in soup.findAll('td',{'class':'ffh'}) + \
soup.findAll('div',{'class':'acl'}) + \
soup.findAll('div',{'class':'adWrap'}) + \
soup.findAll('div',{'class':'footer smtxt'}) + \
soup.findAll('table',{'class':'tbbrdr'}):
tag.extract() # remove tag from soup.
+11 -209
View File
@@ -16,222 +16,24 @@
#
# Software: eFiction
import time
import logging
logger = logging.getLogger(__name__)
import re
import urllib2
from .. import BeautifulSoup as bs
from ..htmlcleanup import stripHTML
from .. import exceptions as exceptions
from base_adapter import BaseSiteAdapter, makeDate
def getClass():
return NHAMagicalWorldsUsAdapter
from base_efiction_adapter import BaseEfictionAdapter
# Class name has to be unique. Our convention is camel case the
# sitename with Adapter at the end. www is skipped.
class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
def __init__(self, config, url):
BaseSiteAdapter.__init__(self, config, url)
self.decode = ["Windows-1252",
"utf8"] # 1252 is a superset of iso-8859-1.
# Most sites that claim to be
# iso-8859-1 (and some that claim to be
# utf8) are really windows-1252.
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
self.password = ""
self.is_adult=False
# get storyId from url--url validation guarantees query is only sid=1234
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
# normalized story URL.
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
# Each adapter needs to have a unique site abbreviation.
self.story.setMetadata('siteabbrev','nha')
# The date format will vary from site to site.
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
self.dateformat = " %d/%m/%y"
@staticmethod # must be @staticmethod, don't remove it.
@staticmethod
def getSiteDomain():
# The site domain. Does have www here, if it uses it.
return 'nha.magical-worlds.us'
@classmethod
def getSiteExampleURLs(cls):
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
def getSiteURLPattern(self):
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
## Getting the chapter list and the meta data, plus 'is adult' checking.
def extractChapterUrlsAndMetadata(self):
url = self.url
logger.debug("URL: "+url)
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
raise e
m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
if m != None:
if self.is_adult or self.getConfig("is_adult"):
# We tried the default and still got a warning, so
# let's pull the warning number from the 'continue'
# link and reload data.
addurl = m.group(1)
# correct stupid &amp; error in url.
addurl = addurl.replace("&amp;","&")
url = self.url+'&index=1'+addurl
logger.debug("URL 2nd try: "+url)
try:
data = self._fetchUrl(url)
except urllib2.HTTPError, e:
if e.code == 404:
raise exceptions.StoryDoesNotExist(self.url)
else:
raise e
else:
raise exceptions.AdultCheckRequired(self.url)
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
# use BeautifulSoup HTML parser to make everything easier to find.
soup = bs.BeautifulSoup(data)
# print data
# Now go hunting for all the meta data and the chapter list.
# Find authorid and URL from... author url.
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
self.story.setMetadata('authorId',a['href'].split('=')[1])
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
self.story.setMetadata('author',a.string)
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
try:
# in case link points somewhere other than the first chapter
a = soup.findAll('option')[1]['value']
self.story.setMetadata('storyId',a.split('=',)[1])
url = 'http://'+self.host+'/'+a
soup = bs.BeautifulSoup(self._fetchUrl(url))
except:
pass
for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
a = info.find('a')
if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
self.story.setMetadata('title',stripHTML(a))
break
# Find the chapters:
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
if len(chapters) == 0:
self.chapterUrls.append((self.story.getMetadata('title'),url))
else:
for chapter in chapters:
# just in case there's tags, like <i> in chapter titles.
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
self.story.setMetadata('numChapters',len(self.chapterUrls))
# eFiction sites don't help us out a lot with their meta data
# formating, so it's a little ugly.
# utility method
def defaultGetattr(d):
try:
return d.name
except:
return ""
cats = info.findAll('a',href=re.compile('categories.php'))
for cat in cats:
self.story.addToList('category',cat.string)
a = info.find('a', href=re.compile(r'viewuser.php'))
val = a.nextSibling
svalue = ""
while not defaultGetattr(val) == 'br':
val = val.nextSibling
val = val.nextSibling
while not defaultGetattr(val) == 'br':
svalue += unicode(val)
val = val.nextSibling
self.setDescription(url,svalue)
def getSiteAbbrev(self):
return 'nha'
#does not provide convenient way to get word count
labels = info.findAll('i')
for labelspan in labels:
value = labelspan.nextSibling
label = stripHTML(labelspan)
if 'Rating' in label:
self.story.setMetadata('rating', value.split(' -')[0])
if 'Genres' in label:
genres = value.string.split(', ')
for genre in genres:
if 'None' not in genre:
self.story.addToList('genre',genre.split(' -')[0])
if 'Characters' in label:
chars = value.string.split(', ')
for char in chars:
if 'None' not in char:
self.story.addToList('characters',char.split(' -')[0])
if 'Warnings' in label:
warnings = value.string.split(', ')
for warning in warnings:
if 'None' not in warning:
self.story.addToList('warnings',warning.split(' -')[0])
if 'Completed' in label:
if 'Yes' in value:
self.story.setMetadata('status', 'Completed')
else:
self.story.setMetadata('status', 'In-Progress')
if 'Published' in label:
self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
if 'Updated' in label:
# there's a stray [ at the end.
#value = value[0:-1]
self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
# grab the text for an individual chapter.
def getChapterText(self, url):
logger.debug('Getting chapter text from: %s' % url)
data = self._fetchUrl(url)
soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
story = soup.find('div', {"id" : "story"})
if None == story:
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
@classmethod
def getDateFormat(self):
return "%d/%m/%y"
def getClass():
return NHAMagicalWorldsUsAdapter
return self.utf8FromSoup(url,story)
+1 -2
View File
@@ -264,7 +264,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
else:
super(NameOfMyAdapter, self).handleMetadata(key, value)
"""
# logger.debug("metadata: '%s' == '%s'" % (key, value))
if value == 'None':
return
elif key == 'Summary':
@@ -287,7 +286,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
self.story.addToList('challenge', val)
elif key == 'Chapters':
self.story.setMetadata('numChapters', int(value))
elif key == 'Rating':
elif key == 'Rating' or key == 'Rated':
self.story.setMetadata('rating', value)
elif key == 'Word count':
self.story.setMetadata('numWords', value)
+5 -40
View File
@@ -327,6 +327,11 @@ sort_ships:false
## User-agent
user_agent:FFF/2.X
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
## Each output format has a section that overrides [defaults]
[html]
@@ -770,11 +775,6 @@ extraships:Spike/Buffy
## this should go in your personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[dramione.org]
## Some sites require login (or login for some rated stories) The
## program can prompt you, or you can save it in config. In
@@ -937,11 +937,6 @@ extraships:Harry Potter/Hermione Granger
#username:YourName
#password:yourpassword
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings,romance
extra_titlepage_entries: readings,romance
readings_label: Readings
@@ -1089,11 +1084,6 @@ extracategories:Glee RPF
extracharacters:Darren Criss, Chris Colfer
extraships:Darren Criss/Chris Colfer
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[ksarchive.com]
## Site dedicated to these categories/characters/ships
extracategories:Star Trek
@@ -1113,11 +1103,6 @@ eroticatags_label:Erotica Tags
extra_titlepage_entries: eroticatags
[lotrfanfiction.com]
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings
readings_label: Readings
@@ -1367,11 +1352,6 @@ extracategories:Transgender
## confirm they are adult for adult content.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[thehexfiles.net]
## Site dedicated to these categories/characters/ships
extracategories:Harry Potter
@@ -1391,11 +1371,6 @@ extraships:Harry Potter/Draco Malfoy
## personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
## Site dedicated to these categories/characters/ships
extracategories:Criminal Minds
@@ -1405,11 +1380,6 @@ extracategories:Criminal Minds
## personal.ini, not defaults.ini.
#is_adult:true
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
extra_valid_entries: readings,challenge
extra_titlepage_entries: readings,challenge
challenge_label: Challenge
@@ -1757,11 +1727,6 @@ extraships:InuYasha/Kagome
## Site dedicated to these categories/characters/ships
extracategories:Lord of the Rings
## Virtually all eFiction Base adapters allow downloading the whole story in
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
## metadata and chapters can be loaded in one step
bulk_load:true
[www.mediaminer.org]
[www.midnightwhispers.ca]
+34 -24
View File
@@ -22,6 +22,9 @@ import re
import urllib2 as u2
import urlparse
import logging
logger = logging.getLogger(__name__)
from BeautifulSoup import BeautifulSoup
from gziphttp import GZipProcessor
@@ -74,7 +77,7 @@ def get_urls_from_page(url,configuration=None,normalize=False):
return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
urls = collections.defaultdict(list)
urls = collections.OrderedDict()
if not configuration:
configuration = Configuration("test1.com","EPUB")
@@ -82,17 +85,17 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
soup = BeautifulSoup(data)
if restrictsearch:
soup = soup.find(*restrictsearch)
#print("restrict search:%s"%soup)
#logger.debug("restrict search:%s"%soup)
for a in soup.findAll('a'):
if a.has_key('href'):
#print("a['href']:%s"%a['href'])
#logger.debug("a['href']:%s"%a['href'])
href = form_url(url,a['href'])
#print("1 urlhref:%s"%href)
#logger.debug("1 urlhref:%s"%href)
# this (should) catch normal story links, some javascript
# 'are you old enough' links, and 'Report This' links.
if 'story.php' in a['href']:
#print("trying:%s"%a['href'])
#logger.debug("trying:%s"%a['href'])
m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
if m != None:
href = form_url(a['href'] if '//' in a['href'] else url,
@@ -100,12 +103,15 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
try:
href = href.replace('&index=1','')
#print("2 urlhref:%s"%href)
#logger.debug("2 urlhref:%s"%href)
adapter = adapters.getAdapter(configuration,href)
#print("found adapter")
urls[adapter.story.getMetadata('storyUrl')].append(href)
#logger.debug("found adapter")
if adapter.story.getMetadata('storyUrl') not in urls:
urls[adapter.story.getMetadata('storyUrl')] = [href]
else:
urls[adapter.story.getMetadata('storyUrl')].append(href)
except Exception, e:
#print e
#logger.debug e
pass
# Simply return the longest URL with the assumption that it contains the
@@ -114,7 +120,7 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
def get_urls_from_text(data,configuration=None,normalize=False):
urls = collections.defaultdict(list)
urls = collections.OrderedDict()
data=unicode(data)
if not configuration:
@@ -130,7 +136,10 @@ def get_urls_from_text(data,configuration=None,normalize=False):
try:
href = href.replace('&index=1','')
adapter = adapters.getAdapter(configuration,href)
urls[adapter.story.getMetadata('storyUrl')].append(href)
if adapter.story.getMetadata('storyUrl') not in urls:
urls[adapter.story.getMetadata('storyUrl')] = [href]
else:
urls[adapter.story.getMetadata('storyUrl')].append(href)
except:
pass
@@ -167,7 +176,8 @@ def form_url(parenturl,url):
return returl
def get_urls_from_imap(srv,user,passwd,folder,markread=True):
logger.debug("get_urls_from_imap srv:(%s)"%srv)
mail = imaplib.IMAP4_SSL(srv)
mail.login(user, passwd)
mail.list()
@@ -176,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
result, data = mail.uid('search', None, "UNSEEN")
#print("result:%s"%result)
#print("data:%s"%data)
#logger.debug("result:%s"%result)
#logger.debug("data:%s"%data)
urls=set()
#latest_email_uid = data[0].split()[-1]
@@ -185,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
#print("result:%s"%result)
#print("data:%s"%data)
#logger.debug("result:%s"%result)
#logger.debug("data:%s"%data)
raw_email = data[0][1]
@@ -195,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
email_message = email.message_from_string(raw_email)
#print "To:%s"%email_message['To']
#print "From:%s"%email_message['From']
#print "Subject:%s"%email_message['Subject']
#logger.debug "To:%s"%email_message['To']
#logger.debug "From:%s"%email_message['From']
#logger.debug "Subject:%s"%email_message['Subject']
# print("payload:%s"%email_message.get_payload())
# logger.debug("payload:%s"%email_message.get_payload())
urllist=[]
for part in email_message.walk():
try:
#print("part mime:%s"%part.get_content_type())
#logger.debug("part mime:%s"%part.get_content_type())
if part.get_content_type() == 'text/plain':
urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
if part.get_content_type() == 'text/html':
urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
except Exception as e:
print("Failed to read email content: %s"%e)
#print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
logger.error("Failed to read email content: %s"%e)
#logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
if urllist and markread:
#obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
#print("seen result:%s->%s"%(email_uid,r))
#logger.debug("seen result:%s->%s"%(email_uid,r))
[ urls.add(x) for x in urllist ]
+5 -1
View File
@@ -781,7 +781,11 @@ class Story(Configurable):
retlist = filter( lambda x : x!=None and x!='' ,retlist)
if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category')) > 1:
if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
removeallentities=False,
# to avoid inf loops if genre/cat substs
doreplacements=False
)) > 1:
retlist.append(self.getConfig('add_genre_when_multi_category'))
# reorder ships so b/a and c/b/a become a/b and a/b/c. Only on '/',
+1 -1
View File
@@ -25,7 +25,7 @@ setup(
# Versions should comply with PEP440. For a discussion on single-sourcing
# the version across setup.py and the project code, see
# https://packaging.python.org/en/latest/single_source_version.html
version="2.2.7",
version="2.2.10",
description='A tool for downloading fanfiction to eBook formats',
long_description=long_description,
+1 -1
View File
@@ -1,6 +1,6 @@
# ffd-retief-hrd fanfictiondownloader fanficfare
application: fanficfare
version: 2-2-7
version: 2-2-10
runtime: python27
api_version: 1
threadsafe: true
+2 -2
View File
@@ -46,7 +46,7 @@
</p>
<h3>Changes:</h3>
<ul>
<li>Fix add_genre_when_multi_category when genre is empty.</li>
<li>Updates for mediaminer.org changes.</li>
</ul>
<p>
Questions? Check out our
@@ -56,7 +56,7 @@
If you have any problems with this application, please
report them in
the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>. The
<a href="http://2-2-6.fanficfare.appspot.com">previous version
<a href="http://2-2-9.fanficfare.appspot.com">previous version
</a> is also available for you to use if necessary.
</p>
<div id='error'>