mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-13 12:11:20 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cae4a74578 | ||
|
|
3839bba182 | ||
|
|
4338f4e550 | ||
|
|
883f9e22cc | ||
|
|
9b468da598 | ||
|
|
fa90a3f23c | ||
|
|
4f7fd93b64 | ||
|
|
d37dadf972 | ||
|
|
5dbbc2efe5 | ||
|
|
c442feeb26 | ||
|
|
17b0800242 | ||
|
|
3c6a60f001 | ||
|
|
96529571b2 | ||
|
|
ab2eb447e2 | ||
|
|
71a44e4e64 | ||
|
|
ff42cd86e2 | ||
|
|
e2c34eaea1 | ||
|
|
78e5d8427b | ||
|
|
c0adf8e027 | ||
|
|
69d1ce6c01 | ||
|
|
390c661a88 |
@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
|
||||
description = _('UI plugin to download FanFiction stories from various sites.')
|
||||
supported_platforms = ['windows', 'osx', 'linux']
|
||||
author = 'Jim Miller'
|
||||
version = (2, 2, 7)
|
||||
version = (2, 2, 10)
|
||||
minimum_calibre_version = (1, 48, 0)
|
||||
|
||||
#: This field defines the GUI plugin class that contains all the code
|
||||
|
||||
@@ -259,6 +259,7 @@ class ConfigWidget(QWidget):
|
||||
prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
|
||||
prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
|
||||
prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
|
||||
prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
|
||||
prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
|
||||
prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
|
||||
|
||||
@@ -280,6 +281,8 @@ class ConfigWidget(QWidget):
|
||||
# if they've removed everything, reset to default.
|
||||
prefs['personal.ini'] = get_resources('plugin-example.ini')
|
||||
|
||||
prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
|
||||
|
||||
# Covers tab
|
||||
prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
|
||||
# for backward compatibility:
|
||||
@@ -321,6 +324,8 @@ class ConfigWidget(QWidget):
|
||||
colsnewonly[col] = checkbox.isChecked()
|
||||
prefs['std_cols_newonly'] = colsnewonly
|
||||
|
||||
prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
|
||||
|
||||
# Custom Columns tab
|
||||
# error column
|
||||
prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
|
||||
@@ -344,10 +349,10 @@ class ConfigWidget(QWidget):
|
||||
|
||||
prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
|
||||
|
||||
prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
|
||||
prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
|
||||
prefs['imappass'] = unicode(self.imap_tab.imappass.text())
|
||||
prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
|
||||
prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
|
||||
prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
|
||||
prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
|
||||
prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
|
||||
prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
|
||||
prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
|
||||
prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
|
||||
@@ -517,6 +522,11 @@ class BasicTab(QWidget):
|
||||
self.injectseries.setChecked(prefs['injectseries'])
|
||||
self.l.addWidget(self.injectseries)
|
||||
|
||||
self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
|
||||
self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier. But if not found, it can also search existing books by Title and Author(s)."))
|
||||
self.matchtitleauth.setChecked(prefs['matchtitleauth'])
|
||||
self.l.addWidget(self.matchtitleauth)
|
||||
|
||||
rej_gb = groupbox = QGroupBox(_("Reject List"))
|
||||
self.l = QVBoxLayout()
|
||||
groupbox.setLayout(self.l)
|
||||
@@ -655,8 +665,13 @@ class PersonalIniTab(QWidget):
|
||||
self.defaults.clicked.connect(self.show_defaults)
|
||||
self.l.addWidget(self.defaults)
|
||||
|
||||
self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
|
||||
self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
|
||||
self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
|
||||
self.l.addWidget(self.cal_cols_pass_in)
|
||||
|
||||
self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
|
||||
self.showcalcols.setToolTip(_("FanFicFare passes the Calibre columns into the download/update process. This will show you the columns available by name."))
|
||||
self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
|
||||
self.showcalcols.clicked.connect(self.show_showcalcols)
|
||||
self.l.addWidget(self.showcalcols)
|
||||
|
||||
@@ -1223,7 +1238,6 @@ class CustomColumnsTab(QWidget):
|
||||
self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
|
||||
self.l.addWidget(self.allow_custcol_from_ini)
|
||||
|
||||
self.l.addSpacing(5)
|
||||
label = QLabel(_("Special column:"))
|
||||
label.setWordWrap(True)
|
||||
self.l.addWidget(label)
|
||||
@@ -1305,6 +1319,17 @@ class StandardColumnsTab(QWidget):
|
||||
horz.addWidget(newonlycheck)
|
||||
|
||||
self.l.addLayout(horz)
|
||||
|
||||
self.l.addSpacing(5)
|
||||
label = QLabel(_("Other Standard Column Options"))
|
||||
label.setWordWrap(True)
|
||||
self.l.addWidget(label)
|
||||
self.l.addSpacing(5)
|
||||
|
||||
self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
|
||||
self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
|
||||
self.set_author_url.setChecked(prefs['set_author_url'])
|
||||
self.l.addWidget(self.set_author_url)
|
||||
|
||||
self.l.insertStretch(-1)
|
||||
|
||||
|
||||
@@ -828,6 +828,7 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
options['version'] = self.version
|
||||
logger.debug(self.version)
|
||||
options['personal.ini'] = get_fff_personalini()
|
||||
options['savemetacol'] = prefs['savemetacol']
|
||||
|
||||
#print("prep_downloads:%s"%books)
|
||||
|
||||
@@ -1023,8 +1024,9 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
|
||||
# all_metadata duplicates some data, but also includes extra_entries, etc.
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
# get metadata to save in configured column.
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
if prefs['savemetacol'] != '':
|
||||
# get metadata to save in configured column.
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
book['title'] = story.getMetadata("title", removeallentities=True)
|
||||
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
|
||||
@@ -1072,7 +1074,7 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
# try to find by identifier url or uri first.
|
||||
identicalbooks = self.do_id_search(url)
|
||||
# print("identicalbooks:%s"%identicalbooks)
|
||||
if len(identicalbooks) < 1:
|
||||
if len(identicalbooks) < 1 and prefs['matchtitleauth']:
|
||||
# find dups
|
||||
authlist = story.getList("author", removeallentities=True)
|
||||
mi = MetaInformation(story.getMetadata("title", removeallentities=True),
|
||||
@@ -1202,51 +1204,52 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
#print("calibre_series:%s [%s]"%book['calibre_series'])
|
||||
|
||||
if book['good']: # there shouldn't be any !'good' books at this point.
|
||||
|
||||
|
||||
## Filling calibre_std_* and calibre_cust_* metadata
|
||||
book['calibre_columns']={}
|
||||
# std columns
|
||||
mi = db.get_metadata(book['calibre_id'],index_is_id=True)
|
||||
# book['calibre_columns']['calibre_std_identifiers']=\
|
||||
# {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
|
||||
# 'label':_('Ids')}
|
||||
for k in mi.standard_field_keys():
|
||||
# for k in mi:
|
||||
if k in STD_COLS_SKIP:
|
||||
continue
|
||||
(label,value,v,fmd) = mi.format_field_extended(k)
|
||||
if not label and k in field_metadata:
|
||||
label=field_metadata[k]['name']
|
||||
key='calibre_std_'+k
|
||||
|
||||
# if k == 'user_categories':
|
||||
# value=u', '.join(mi.get(k))
|
||||
# label=_('User Categories')
|
||||
|
||||
if label: # only if it has a human readable name.
|
||||
if value is None or not book['calibre_id']:
|
||||
## if existing book, populate existing calibre column
|
||||
## values in metadata, else '' to hide.
|
||||
value=''
|
||||
book['calibre_columns'][key]={'val':value,'label':label}
|
||||
#logger.debug("%s(%s): %s"%(label,key,value))
|
||||
|
||||
# custom columns
|
||||
for k, column in self.gui.library_view.model().custom_columns.iteritems():
|
||||
if k != prefs['savemetacol']:
|
||||
key='calibre_cust_'+k[1:]
|
||||
label=column['name']
|
||||
value=db.get_custom(book['calibre_id'],
|
||||
label=column['label'],
|
||||
index_is_id=True)
|
||||
# custom always have name.
|
||||
if value is None or not book['calibre_id']:
|
||||
## if existing book, populate existing calibre column
|
||||
## values in metadata, else '' to hide.
|
||||
value=''
|
||||
book['calibre_columns'][key]={'val':value,'label':label}
|
||||
# logger.debug("%s(%s): %s"%(label,key,value))
|
||||
if prefs['cal_cols_pass_in']:
|
||||
# std columns
|
||||
mi = db.get_metadata(book['calibre_id'],index_is_id=True)
|
||||
# book['calibre_columns']['calibre_std_identifiers']=\
|
||||
# {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
|
||||
# 'label':_('Ids')}
|
||||
for k in mi.standard_field_keys():
|
||||
# for k in mi:
|
||||
if k in STD_COLS_SKIP:
|
||||
continue
|
||||
(label,value,v,fmd) = mi.format_field_extended(k)
|
||||
if not label and k in field_metadata:
|
||||
label=field_metadata[k]['name']
|
||||
key='calibre_std_'+k
|
||||
|
||||
# if k == 'user_categories':
|
||||
# value=u', '.join(mi.get(k))
|
||||
# label=_('User Categories')
|
||||
|
||||
if label: # only if it has a human readable name.
|
||||
if value is None or not book['calibre_id']:
|
||||
## if existing book, populate existing calibre column
|
||||
## values in metadata, else '' to hide.
|
||||
value=''
|
||||
book['calibre_columns'][key]={'val':value,'label':label}
|
||||
#logger.debug("%s(%s): %s"%(label,key,value))
|
||||
|
||||
# custom columns
|
||||
for k, column in self.gui.library_view.model().custom_columns.iteritems():
|
||||
if k != prefs['savemetacol']:
|
||||
key='calibre_cust_'+k[1:]
|
||||
label=column['name']
|
||||
value=db.get_custom(book['calibre_id'],
|
||||
label=column['label'],
|
||||
index_is_id=True)
|
||||
# custom always have name.
|
||||
if value is None or not book['calibre_id']:
|
||||
## if existing book, populate existing calibre column
|
||||
## values in metadata, else '' to hide.
|
||||
value=''
|
||||
book['calibre_columns'][key]={'val':value,'label':label}
|
||||
# logger.debug("%s(%s): %s"%(label,key,value))
|
||||
|
||||
# if still 'good', make a temp file to write the output to.
|
||||
# For HTML format users, make the filename inside the zip something reasonable.
|
||||
# For crazy long titles/authors, limit it to 200chars.
|
||||
@@ -1738,7 +1741,7 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
# save metadata to configured column
|
||||
if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
|
||||
label = custom_columns[prefs['savemetacol']]['label']
|
||||
self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata()
|
||||
self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
|
||||
|
||||
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])
|
||||
for col, meta in prefs['custom_cols'].iteritems():
|
||||
@@ -1840,7 +1843,7 @@ class FanFicFarePlugin(InterfaceAction):
|
||||
|
||||
# set author link if found. All current adapters have authorUrl, except anonymous on AO3.
|
||||
# Moved down so author's already in the DB.
|
||||
if 'authorUrl' in book['all_metadata']:
|
||||
if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
|
||||
authurls = book['all_metadata']['authorUrl'].split(", ")
|
||||
authorlist = [ a.replace('&',';') for a in book['author'] ]
|
||||
authorids = db.new_api.get_item_ids('authors',authorlist)
|
||||
|
||||
@@ -156,7 +156,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
||||
logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
|
||||
book['comment'] = 'Metadata collected.'
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
## checks were done earlier, it's new or not dup or newer--just write it.
|
||||
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
|
||||
@@ -176,7 +177,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
||||
writer.writeStory(outfilename=outfile, forceOverwrite=True)
|
||||
book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
## checks were done earlier, just update it.
|
||||
elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
|
||||
@@ -197,7 +199,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
||||
if chaptercount == urlchaptercount:
|
||||
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
|
||||
return book
|
||||
|
||||
@@ -218,8 +221,9 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
|
||||
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
|
||||
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
|
||||
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
if options['savemetacol'] != '':
|
||||
book['savemetacol'] = story.dump_html_metadata()
|
||||
|
||||
if options['smarten_punctuation'] and options['fileform'] == "epub" \
|
||||
and calibre_version >= (0, 9, 39):
|
||||
# for smarten punc
|
||||
|
||||
@@ -330,6 +330,11 @@ sort_ships:false
|
||||
## User-agent
|
||||
user_agent:FFF/2.X
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
## Each output format has a section that overrides [defaults]
|
||||
[html]
|
||||
|
||||
@@ -766,11 +771,6 @@ extraships:Spike/Buffy
|
||||
## this should go in your personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[dramione.org]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
@@ -951,11 +951,6 @@ extraships:Harry Potter/Hermione Granger
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings,romance
|
||||
extra_titlepage_entries: readings,romance
|
||||
readings_label: Readings
|
||||
@@ -1103,11 +1098,6 @@ extracategories:Glee RPF
|
||||
extracharacters:Darren Criss, Chris Colfer
|
||||
extraships:Darren Criss/Chris Colfer
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[ksarchive.com]
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Star Trek
|
||||
@@ -1127,11 +1117,6 @@ eroticatags_label:Erotica Tags
|
||||
extra_titlepage_entries: eroticatags
|
||||
|
||||
[lotrfanfiction.com]
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings
|
||||
readings_label: Readings
|
||||
|
||||
@@ -1381,11 +1366,6 @@ extracategories:Transgender
|
||||
## confirm they are adult for adult content.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[thehexfiles.net]
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Harry Potter
|
||||
@@ -1405,11 +1385,6 @@ extraships:Harry Potter/Draco Malfoy
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Criminal Minds
|
||||
|
||||
@@ -1419,11 +1394,6 @@ extracategories:Criminal Minds
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings,challenge
|
||||
extra_titlepage_entries: readings,challenge
|
||||
challenge_label: Challenge
|
||||
@@ -1777,11 +1747,6 @@ extraships:InuYasha/Kagome
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Lord of the Rings
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[www.mediaminer.org]
|
||||
|
||||
[www.midnightwhispers.ca]
|
||||
|
||||
@@ -51,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
|
||||
# take from here.
|
||||
default_prefs = {}
|
||||
default_prefs['personal.ini'] = get_resources('plugin-example.ini')
|
||||
default_prefs['cal_cols_pass_in'] = False
|
||||
default_prefs['rejecturls'] = ''
|
||||
default_prefs['rejectreasons'] = '''Sucked
|
||||
Boring
|
||||
@@ -75,6 +76,7 @@ default_prefs['lookforurlinhtml'] = False
|
||||
default_prefs['checkforseriesurlid'] = True
|
||||
default_prefs['checkforurlchange'] = True
|
||||
default_prefs['injectseries'] = False
|
||||
default_prefs['matchtitleauth'] = True
|
||||
default_prefs['smarten_punctuation'] = False
|
||||
default_prefs['show_est_time'] = False
|
||||
|
||||
@@ -104,6 +106,7 @@ default_prefs['custom_cols_newonly'] = {}
|
||||
default_prefs['allow_custcol_from_ini'] = True
|
||||
|
||||
default_prefs['std_cols_newonly'] = {}
|
||||
default_prefs['set_author_url'] = True
|
||||
|
||||
default_prefs['imapserver'] = ''
|
||||
default_prefs['imapuser'] = ''
|
||||
|
||||
+363
-329
File diff suppressed because it is too large
Load Diff
+372
-338
File diff suppressed because it is too large
Load Diff
+359
-325
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+356
-322
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+355
-321
File diff suppressed because it is too large
Load Diff
@@ -22,7 +22,6 @@ import re
|
||||
import urllib
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
# get storyId from url--url validation guarantees query correct
|
||||
m = re.match(self.getSiteURLPattern(),url)
|
||||
if m:
|
||||
self.story.setMetadata('storyId',m.group('id'))
|
||||
if m.group('id'):
|
||||
self.story.setMetadata('storyId',m.group('id'))
|
||||
elif m.group('id2'):
|
||||
self.story.setMetadata('storyId',m.group('id2'))
|
||||
elif m.group('id3'):
|
||||
self.story.setMetadata('storyId',m.group('id2'))
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
|
||||
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
def getSiteURLPattern(self):
|
||||
## http://www.mediaminer.org/fanfic/view_st.php/76882
|
||||
## http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
|
||||
## http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
|
||||
## http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
|
||||
return re.escape("http://"+self.getSiteDomain())+\
|
||||
"/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
|
||||
r"/fanfic/view_(st|ch)\.php"+\
|
||||
r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
|
||||
r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
|
||||
|
||||
# Override stripURLParameters so the id parameter won't get stripped
|
||||
@classmethod
|
||||
def stripURLParameters(cls, url):
|
||||
return url
|
||||
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
|
||||
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
raise e
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
soup = self.make_soup(data)
|
||||
|
||||
# [ A - All Readers ], strip '[' ']'
|
||||
## Above title because we remove the smtxt font to get title.
|
||||
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
title = soup.find('td',{'class':'ffh'})
|
||||
for font in title.findAll('font'):
|
||||
font.extract() # removes 'font' tags from inside the td.
|
||||
if title.has_key('colspan'):
|
||||
if title.has_attr('colspan'):
|
||||
titlet = stripHTML(title)
|
||||
else:
|
||||
## No colspan, it's part chapter title--even if it's a one-shot.
|
||||
titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
|
||||
self.story.setMetadata('title',titlet)
|
||||
## The story title is difficult to reliably parse from the
|
||||
## story pages. Getting it from the author page is, but costs
|
||||
## another fetch.
|
||||
# authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
# titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
|
||||
# self.story.setMetadata('title',titlea.text)
|
||||
|
||||
# save date from first for later.
|
||||
firstdate=None
|
||||
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
# save date from first for later.
|
||||
if not firstdate:
|
||||
firstdate = m.group(3)
|
||||
self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
|
||||
# http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
|
||||
# self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
|
||||
self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# category
|
||||
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
data=self._fetchUrl(url)
|
||||
soup = bs.BeautifulStoneSoup(data,
|
||||
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
|
||||
soup = self.make_soup(data)
|
||||
|
||||
anchor = soup.find('a',{'name':'fic_c'})
|
||||
header = soup.find('div',{'class':'post-meta clearfix '})
|
||||
# print("data:%s"%data)
|
||||
|
||||
if None == anchor:
|
||||
chapter=self.make_soup('<div class="story"></div>').find('div')
|
||||
|
||||
if None == header:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
|
||||
## find divs with align=left, those are paragraphs in newer stories.
|
||||
divlist = anchor.findAllNext('div',{'align':'left'})
|
||||
divlist = header.findAllNext('div',{'align':'left'})
|
||||
if divlist:
|
||||
for div in divlist:
|
||||
div.name='p' # convert to <p> mediaminer uses div with
|
||||
# a margin for paragraphs.
|
||||
anchor.append(div) # cheat! stuff all the content
|
||||
# divs into anchor just as a
|
||||
# holder.
|
||||
chapter.append(div)
|
||||
del div['style']
|
||||
del div['align']
|
||||
anchor.name='div'
|
||||
return self.utf8FromSoup(url,anchor)
|
||||
return self.utf8FromSoup(url,chapter)
|
||||
|
||||
else:
|
||||
logger.debug('Using kludgey text find for older mediaminer story.')
|
||||
## Some older mediaminer stories are unparsable with BeautifulSoup.
|
||||
## Really nasty formatting. Sooo... Cheat! Parse it ourselves a bit first.
|
||||
## Story stuff falls between:
|
||||
data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
|
||||
soup = bs.BeautifulStoneSoup(data,
|
||||
selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
|
||||
data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
|
||||
soup = self.make_soup(data)
|
||||
for tag in soup.findAll('td',{'class':'ffh'}) + \
|
||||
soup.findAll('div',{'class':'acl'}) + \
|
||||
soup.findAll('div',{'class':'adWrap'}) + \
|
||||
soup.findAll('div',{'class':'footer smtxt'}) + \
|
||||
soup.findAll('table',{'class':'tbbrdr'}):
|
||||
tag.extract() # remove tag from soup.
|
||||
|
||||
@@ -16,222 +16,24 @@
|
||||
#
|
||||
|
||||
# Software: eFiction
|
||||
import time
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
import re
|
||||
import urllib2
|
||||
|
||||
from .. import BeautifulSoup as bs
|
||||
from ..htmlcleanup import stripHTML
|
||||
from .. import exceptions as exceptions
|
||||
|
||||
from base_adapter import BaseSiteAdapter, makeDate
|
||||
|
||||
def getClass():
|
||||
return NHAMagicalWorldsUsAdapter
|
||||
from base_efiction_adapter import BaseEfictionAdapter
|
||||
|
||||
# Class name has to be unique. Our convention is camel case the
|
||||
# sitename with Adapter at the end. www is skipped.
|
||||
class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
|
||||
class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
|
||||
|
||||
def __init__(self, config, url):
|
||||
BaseSiteAdapter.__init__(self, config, url)
|
||||
|
||||
self.decode = ["Windows-1252",
|
||||
"utf8"] # 1252 is a superset of iso-8859-1.
|
||||
# Most sites that claim to be
|
||||
# iso-8859-1 (and some that claim to be
|
||||
# utf8) are really windows-1252.
|
||||
self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
|
||||
self.password = ""
|
||||
self.is_adult=False
|
||||
|
||||
# get storyId from url--url validation guarantees query is only sid=1234
|
||||
self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
|
||||
|
||||
|
||||
|
||||
# normalized story URL.
|
||||
self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
|
||||
|
||||
# Each adapter needs to have a unique site abbreviation.
|
||||
self.story.setMetadata('siteabbrev','nha')
|
||||
|
||||
# The date format will vary from site to site.
|
||||
# http://docs.python.org/library/datetime.html#strftime-strptime-behavior
|
||||
self.dateformat = " %d/%m/%y"
|
||||
|
||||
@staticmethod # must be @staticmethod, don't remove it.
|
||||
@staticmethod
|
||||
def getSiteDomain():
|
||||
# The site domain. Does have www here, if it uses it.
|
||||
return 'nha.magical-worlds.us'
|
||||
|
||||
@classmethod
|
||||
def getSiteExampleURLs(cls):
|
||||
return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
|
||||
|
||||
def getSiteURLPattern(self):
|
||||
return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
|
||||
|
||||
## Getting the chapter list and the meta data, plus 'is adult' checking.
|
||||
def extractChapterUrlsAndMetadata(self):
|
||||
url = self.url
|
||||
logger.debug("URL: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
|
||||
m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
|
||||
if m != None:
|
||||
if self.is_adult or self.getConfig("is_adult"):
|
||||
# We tried the default and still got a warning, so
|
||||
# let's pull the warning number from the 'continue'
|
||||
# link and reload data.
|
||||
addurl = m.group(1)
|
||||
# correct stupid & error in url.
|
||||
addurl = addurl.replace("&","&")
|
||||
url = self.url+'&index=1'+addurl
|
||||
logger.debug("URL 2nd try: "+url)
|
||||
|
||||
try:
|
||||
data = self._fetchUrl(url)
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
raise exceptions.StoryDoesNotExist(self.url)
|
||||
else:
|
||||
raise e
|
||||
else:
|
||||
raise exceptions.AdultCheckRequired(self.url)
|
||||
|
||||
if "Access denied. This story has not been validated by the adminstrators of this site." in data:
|
||||
raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
|
||||
|
||||
# use BeautifulSoup HTML parser to make everything easier to find.
|
||||
soup = bs.BeautifulSoup(data)
|
||||
# print data
|
||||
|
||||
# Now go hunting for all the meta data and the chapter list.
|
||||
|
||||
# Find authorid and URL from... author url.
|
||||
a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
|
||||
self.story.setMetadata('authorId',a['href'].split('=')[1])
|
||||
self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
|
||||
self.story.setMetadata('author',a.string)
|
||||
asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
|
||||
|
||||
try:
|
||||
# in case link points somewhere other than the first chapter
|
||||
a = soup.findAll('option')[1]['value']
|
||||
self.story.setMetadata('storyId',a.split('=',)[1])
|
||||
url = 'http://'+self.host+'/'+a
|
||||
soup = bs.BeautifulSoup(self._fetchUrl(url))
|
||||
except:
|
||||
pass
|
||||
|
||||
for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
|
||||
a = info.find('a')
|
||||
if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
|
||||
('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
|
||||
self.story.setMetadata('title',stripHTML(a))
|
||||
break
|
||||
|
||||
|
||||
# Find the chapters:
|
||||
chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
|
||||
if len(chapters) == 0:
|
||||
self.chapterUrls.append((self.story.getMetadata('title'),url))
|
||||
else:
|
||||
for chapter in chapters:
|
||||
# just in case there's tags, like <i> in chapter titles.
|
||||
self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
|
||||
|
||||
self.story.setMetadata('numChapters',len(self.chapterUrls))
|
||||
|
||||
# eFiction sites don't help us out a lot with their meta data
|
||||
# formating, so it's a little ugly.
|
||||
|
||||
# utility method
|
||||
def defaultGetattr(d):
|
||||
try:
|
||||
return d.name
|
||||
except:
|
||||
return ""
|
||||
|
||||
cats = info.findAll('a',href=re.compile('categories.php'))
|
||||
for cat in cats:
|
||||
self.story.addToList('category',cat.string)
|
||||
|
||||
a = info.find('a', href=re.compile(r'viewuser.php'))
|
||||
val = a.nextSibling
|
||||
svalue = ""
|
||||
while not defaultGetattr(val) == 'br':
|
||||
val = val.nextSibling
|
||||
val = val.nextSibling
|
||||
while not defaultGetattr(val) == 'br':
|
||||
svalue += unicode(val)
|
||||
val = val.nextSibling
|
||||
self.setDescription(url,svalue)
|
||||
def getSiteAbbrev(self):
|
||||
return 'nha'
|
||||
|
||||
#does not provide convenient way to get word count
|
||||
labels = info.findAll('i')
|
||||
for labelspan in labels:
|
||||
value = labelspan.nextSibling
|
||||
label = stripHTML(labelspan)
|
||||
|
||||
if 'Rating' in label:
|
||||
self.story.setMetadata('rating', value.split(' -')[0])
|
||||
|
||||
if 'Genres' in label:
|
||||
genres = value.string.split(', ')
|
||||
for genre in genres:
|
||||
if 'None' not in genre:
|
||||
self.story.addToList('genre',genre.split(' -')[0])
|
||||
|
||||
if 'Characters' in label:
|
||||
chars = value.string.split(', ')
|
||||
for char in chars:
|
||||
if 'None' not in char:
|
||||
self.story.addToList('characters',char.split(' -')[0])
|
||||
|
||||
if 'Warnings' in label:
|
||||
warnings = value.string.split(', ')
|
||||
for warning in warnings:
|
||||
if 'None' not in warning:
|
||||
self.story.addToList('warnings',warning.split(' -')[0])
|
||||
|
||||
if 'Completed' in label:
|
||||
if 'Yes' in value:
|
||||
self.story.setMetadata('status', 'Completed')
|
||||
else:
|
||||
self.story.setMetadata('status', 'In-Progress')
|
||||
|
||||
if 'Published' in label:
|
||||
self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
|
||||
|
||||
if 'Updated' in label:
|
||||
# there's a stray [ at the end.
|
||||
#value = value[0:-1]
|
||||
self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
|
||||
|
||||
|
||||
# grab the text for an individual chapter.
|
||||
def getChapterText(self, url):
|
||||
|
||||
logger.debug('Getting chapter text from: %s' % url)
|
||||
|
||||
data = self._fetchUrl(url)
|
||||
|
||||
soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
|
||||
|
||||
story = soup.find('div', {"id" : "story"})
|
||||
|
||||
if None == story:
|
||||
raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url)
|
||||
@classmethod
|
||||
def getDateFormat(self):
|
||||
return "%d/%m/%y"
|
||||
|
||||
def getClass():
|
||||
return NHAMagicalWorldsUsAdapter
|
||||
|
||||
return self.utf8FromSoup(url,story)
|
||||
|
||||
@@ -264,7 +264,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
|
||||
else:
|
||||
super(NameOfMyAdapter, self).handleMetadata(key, value)
|
||||
"""
|
||||
# logger.debug("metadata: '%s' == '%s'" % (key, value))
|
||||
if value == 'None':
|
||||
return
|
||||
elif key == 'Summary':
|
||||
@@ -287,7 +286,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
|
||||
self.story.addToList('challenge', val)
|
||||
elif key == 'Chapters':
|
||||
self.story.setMetadata('numChapters', int(value))
|
||||
elif key == 'Rating':
|
||||
elif key == 'Rating' or key == 'Rated':
|
||||
self.story.setMetadata('rating', value)
|
||||
elif key == 'Word count':
|
||||
self.story.setMetadata('numWords', value)
|
||||
|
||||
+5
-40
@@ -327,6 +327,11 @@ sort_ships:false
|
||||
## User-agent
|
||||
user_agent:FFF/2.X
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
## Each output format has a section that overrides [defaults]
|
||||
[html]
|
||||
|
||||
@@ -770,11 +775,6 @@ extraships:Spike/Buffy
|
||||
## this should go in your personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[dramione.org]
|
||||
## Some sites require login (or login for some rated stories) The
|
||||
## program can prompt you, or you can save it in config. In
|
||||
@@ -937,11 +937,6 @@ extraships:Harry Potter/Hermione Granger
|
||||
#username:YourName
|
||||
#password:yourpassword
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings,romance
|
||||
extra_titlepage_entries: readings,romance
|
||||
readings_label: Readings
|
||||
@@ -1089,11 +1084,6 @@ extracategories:Glee RPF
|
||||
extracharacters:Darren Criss, Chris Colfer
|
||||
extraships:Darren Criss/Chris Colfer
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[ksarchive.com]
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Star Trek
|
||||
@@ -1113,11 +1103,6 @@ eroticatags_label:Erotica Tags
|
||||
extra_titlepage_entries: eroticatags
|
||||
|
||||
[lotrfanfiction.com]
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings
|
||||
readings_label: Readings
|
||||
|
||||
@@ -1367,11 +1352,6 @@ extracategories:Transgender
|
||||
## confirm they are adult for adult content.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[thehexfiles.net]
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Harry Potter
|
||||
@@ -1391,11 +1371,6 @@ extraships:Harry Potter/Draco Malfoy
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Criminal Minds
|
||||
|
||||
@@ -1405,11 +1380,6 @@ extracategories:Criminal Minds
|
||||
## personal.ini, not defaults.ini.
|
||||
#is_adult:true
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
extra_valid_entries: readings,challenge
|
||||
extra_titlepage_entries: readings,challenge
|
||||
challenge_label: Challenge
|
||||
@@ -1757,11 +1727,6 @@ extraships:InuYasha/Kagome
|
||||
## Site dedicated to these categories/characters/ships
|
||||
extracategories:Lord of the Rings
|
||||
|
||||
## Virtually all eFiction Base adapters allow downloading the whole story in
|
||||
## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
|
||||
## metadata and chapters can be loaded in one step
|
||||
bulk_load:true
|
||||
|
||||
[www.mediaminer.org]
|
||||
|
||||
[www.midnightwhispers.ca]
|
||||
|
||||
+34
-24
@@ -22,6 +22,9 @@ import re
|
||||
import urllib2 as u2
|
||||
import urlparse
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from BeautifulSoup import BeautifulSoup
|
||||
from gziphttp import GZipProcessor
|
||||
|
||||
@@ -74,7 +77,7 @@ def get_urls_from_page(url,configuration=None,normalize=False):
|
||||
return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
|
||||
|
||||
def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
|
||||
urls = collections.defaultdict(list)
|
||||
urls = collections.OrderedDict()
|
||||
|
||||
if not configuration:
|
||||
configuration = Configuration("test1.com","EPUB")
|
||||
@@ -82,17 +85,17 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
|
||||
soup = BeautifulSoup(data)
|
||||
if restrictsearch:
|
||||
soup = soup.find(*restrictsearch)
|
||||
#print("restrict search:%s"%soup)
|
||||
#logger.debug("restrict search:%s"%soup)
|
||||
|
||||
for a in soup.findAll('a'):
|
||||
if a.has_key('href'):
|
||||
#print("a['href']:%s"%a['href'])
|
||||
#logger.debug("a['href']:%s"%a['href'])
|
||||
href = form_url(url,a['href'])
|
||||
#print("1 urlhref:%s"%href)
|
||||
#logger.debug("1 urlhref:%s"%href)
|
||||
# this (should) catch normal story links, some javascript
|
||||
# 'are you old enough' links, and 'Report This' links.
|
||||
if 'story.php' in a['href']:
|
||||
#print("trying:%s"%a['href'])
|
||||
#logger.debug("trying:%s"%a['href'])
|
||||
m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
|
||||
if m != None:
|
||||
href = form_url(a['href'] if '//' in a['href'] else url,
|
||||
@@ -100,12 +103,15 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
|
||||
|
||||
try:
|
||||
href = href.replace('&index=1','')
|
||||
#print("2 urlhref:%s"%href)
|
||||
#logger.debug("2 urlhref:%s"%href)
|
||||
adapter = adapters.getAdapter(configuration,href)
|
||||
#print("found adapter")
|
||||
urls[adapter.story.getMetadata('storyUrl')].append(href)
|
||||
#logger.debug("found adapter")
|
||||
if adapter.story.getMetadata('storyUrl') not in urls:
|
||||
urls[adapter.story.getMetadata('storyUrl')] = [href]
|
||||
else:
|
||||
urls[adapter.story.getMetadata('storyUrl')].append(href)
|
||||
except Exception, e:
|
||||
#print e
|
||||
#logger.debug e
|
||||
pass
|
||||
|
||||
# Simply return the longest URL with the assumption that it contains the
|
||||
@@ -114,7 +120,7 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
|
||||
|
||||
|
||||
def get_urls_from_text(data,configuration=None,normalize=False):
|
||||
urls = collections.defaultdict(list)
|
||||
urls = collections.OrderedDict()
|
||||
data=unicode(data)
|
||||
|
||||
if not configuration:
|
||||
@@ -130,7 +136,10 @@ def get_urls_from_text(data,configuration=None,normalize=False):
|
||||
try:
|
||||
href = href.replace('&index=1','')
|
||||
adapter = adapters.getAdapter(configuration,href)
|
||||
urls[adapter.story.getMetadata('storyUrl')].append(href)
|
||||
if adapter.story.getMetadata('storyUrl') not in urls:
|
||||
urls[adapter.story.getMetadata('storyUrl')] = [href]
|
||||
else:
|
||||
urls[adapter.story.getMetadata('storyUrl')].append(href)
|
||||
except:
|
||||
pass
|
||||
|
||||
@@ -167,7 +176,8 @@ def form_url(parenturl,url):
|
||||
return returl
|
||||
|
||||
def get_urls_from_imap(srv,user,passwd,folder,markread=True):
|
||||
|
||||
|
||||
logger.debug("get_urls_from_imap srv:(%s)"%srv)
|
||||
mail = imaplib.IMAP4_SSL(srv)
|
||||
mail.login(user, passwd)
|
||||
mail.list()
|
||||
@@ -176,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
|
||||
|
||||
result, data = mail.uid('search', None, "UNSEEN")
|
||||
|
||||
#print("result:%s"%result)
|
||||
#print("data:%s"%data)
|
||||
#logger.debug("result:%s"%result)
|
||||
#logger.debug("data:%s"%data)
|
||||
urls=set()
|
||||
|
||||
#latest_email_uid = data[0].split()[-1]
|
||||
@@ -185,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
|
||||
|
||||
result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
|
||||
|
||||
#print("result:%s"%result)
|
||||
#print("data:%s"%data)
|
||||
#logger.debug("result:%s"%result)
|
||||
#logger.debug("data:%s"%data)
|
||||
|
||||
raw_email = data[0][1]
|
||||
|
||||
@@ -195,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
|
||||
|
||||
email_message = email.message_from_string(raw_email)
|
||||
|
||||
#print "To:%s"%email_message['To']
|
||||
#print "From:%s"%email_message['From']
|
||||
#print "Subject:%s"%email_message['Subject']
|
||||
#logger.debug "To:%s"%email_message['To']
|
||||
#logger.debug "From:%s"%email_message['From']
|
||||
#logger.debug "Subject:%s"%email_message['Subject']
|
||||
|
||||
# print("payload:%s"%email_message.get_payload())
|
||||
# logger.debug("payload:%s"%email_message.get_payload())
|
||||
|
||||
urllist=[]
|
||||
for part in email_message.walk():
|
||||
try:
|
||||
#print("part mime:%s"%part.get_content_type())
|
||||
#logger.debug("part mime:%s"%part.get_content_type())
|
||||
if part.get_content_type() == 'text/plain':
|
||||
urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
|
||||
if part.get_content_type() == 'text/html':
|
||||
urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
|
||||
except Exception as e:
|
||||
print("Failed to read email content: %s"%e)
|
||||
#print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
|
||||
logger.error("Failed to read email content: %s"%e)
|
||||
#logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
|
||||
|
||||
if urllist and markread:
|
||||
#obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
|
||||
r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
|
||||
#print("seen result:%s->%s"%(email_uid,r))
|
||||
#logger.debug("seen result:%s->%s"%(email_uid,r))
|
||||
|
||||
[ urls.add(x) for x in urllist ]
|
||||
|
||||
|
||||
+5
-1
@@ -781,7 +781,11 @@ class Story(Configurable):
|
||||
|
||||
retlist = filter( lambda x : x!=None and x!='' ,retlist)
|
||||
|
||||
if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category')) > 1:
|
||||
if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
|
||||
removeallentities=False,
|
||||
# to avoid inf loops if genre/cat substs
|
||||
doreplacements=False
|
||||
)) > 1:
|
||||
retlist.append(self.getConfig('add_genre_when_multi_category'))
|
||||
|
||||
# reorder ships so b/a and c/b/a become a/b and a/b/c. Only on '/',
|
||||
|
||||
@@ -25,7 +25,7 @@ setup(
|
||||
# Versions should comply with PEP440. For a discussion on single-sourcing
|
||||
# the version across setup.py and the project code, see
|
||||
# https://packaging.python.org/en/latest/single_source_version.html
|
||||
version="2.2.7",
|
||||
version="2.2.10",
|
||||
|
||||
description='A tool for downloading fanfiction to eBook formats',
|
||||
long_description=long_description,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# ffd-retief-hrd fanfictiondownloader fanficfare
|
||||
application: fanficfare
|
||||
version: 2-2-7
|
||||
version: 2-2-10
|
||||
runtime: python27
|
||||
api_version: 1
|
||||
threadsafe: true
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
</p>
|
||||
<h3>Changes:</h3>
|
||||
<ul>
|
||||
<li>Fix add_genre_when_multi_category when genre is empty.</li>
|
||||
<li>Updates for mediaminer.org changes.</li>
|
||||
</ul>
|
||||
<p>
|
||||
Questions? Check out our
|
||||
@@ -56,7 +56,7 @@
|
||||
If you have any problems with this application, please
|
||||
report them in
|
||||
the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>. The
|
||||
<a href="http://2-2-6.fanficfare.appspot.com">previous version
|
||||
<a href="http://2-2-9.fanficfare.appspot.com">previous version
|
||||
</a> is also available for you to use if necessary.
|
||||
</p>
|
||||
<div id='error'>
|
||||
|
||||
Reference in New Issue
Block a user