wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-11 11:52:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Save and use metadata from a custom column, and pass cal cols into fff on updates.

Browse Source
This commit is contained in:
Jim Miller
2015-05-08 16:36:08 -05:00
parent 29185ec574
commit ef52acd5b3
10 changed files with 197 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
calibre-plugin/config.py
+22 -3
View File
@@ -323,6 +323,9 @@ class ConfigWidget(QWidget):
# error column
prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
# metadata column
prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
# cust cols tab
colsmap = {}
for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -680,14 +683,15 @@ class PersonalIniTab(QWidget):
self.personalini = d.get_plain_text()
def show_showcalcols(self):
lines=[('calibre_std_user_categories',_('User Categories'))]
lines=[]#[('calibre_std_user_categories',_('User Categories'))]
for k,f in field_metadata.iteritems():
if f['name']: # only if it has a human readable name.
lines.append(('calibre_std_'+k,f['name']))
for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
# custom always have name.
lines.append(('calibre_cust_'+k[1:],column['name']))
if k != prefs['savemetacol']:
# custom always have name.
lines.append(('calibre_cust_'+k[1:],column['name']))
lines.sort() # sort by key.
@@ -1235,6 +1239,21 @@ class CustomColumnsTab(QWidget):
self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
horz.addWidget(self.errorcol)
self.l.addLayout(horz)
horz = QHBoxLayout()
label = QLabel(_("Saved Metadata Column:"))
tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
label.setToolTip(tooltip)
horz.addWidget(label)
self.savemetacol = QComboBox(self)
self.savemetacol.setToolTip(tooltip)
self.savemetacol.addItem('','none')
for key, column in custom_columns.iteritems():
if column['datatype'] in ('comments'):
self.savemetacol.addItem(column['name'],key)
self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
horz.addWidget(self.savemetacol)
self.l.addLayout(horz)
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])
calibre-plugin/dialogs.py
+6 -2
View File
@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
UPDATEALWAYS=_('Update EPUB Always')
OVERWRITE=_('Overwrite if Newer')
OVERWRITEALWAYS=_('Overwrite Always')
CALIBREONLY=_('Update Calibre Metadata Only')
CALIBREONLY=_('Update Calibre Metadata from Web Site')
CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
collision_order=[SKIP,
ADDNEW,
UPDATE,
UPDATEALWAYS,
OVERWRITE,
OVERWRITEALWAYS,
CALIBREONLY,]
CALIBREONLY,
CALIBREONLYSAVECOL,]
# best idea I've had for how to deal with config/pref saving the
# collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
SAVE_OVERWRITE='Overwrite if Newer'
SAVE_OVERWRITEALWAYS='Overwrite Always'
SAVE_CALIBREONLY='Update Calibre Metadata Only'
SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
save_collisions={
SKIP:SAVE_SKIP,
ADDNEW:SAVE_ADDNEW,
@@ -112,6 +115,7 @@ save_collisions={
SAVE_OVERWRITE:OVERWRITE,
SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
SAVE_CALIBREONLY:CALIBREONLY,
SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
}
anthology_collision_order=[UPDATE,
calibre-plugin/fff_plugin.py
+43 -27
View File
@@ -86,6 +86,7 @@ from calibre_plugins.fanficfare_plugin.dialogs import (
LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
RejectListDialog, EmailPassDialog,
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
CALIBREONLYSAVECOL,
NotGoingToDownload, RejectUrlEntry )
# because calibre immediately transforms html into zip and don't want
@@ -902,12 +903,21 @@ class FanFicFarePlugin(InterfaceAction):
options['cookiejar'] = adapter.get_empty_cookiejar()
adapter.set_cookiejar(options['cookiejar'])
## XXX get_epub_metadata works, but how to use it?
if 1==0 and collision in (CALIBREONLY) and \
fileform == 'epub' and \
db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
# adapter.setStoryMetadata(get_epub_metadatas(StringIO(db.format(book['calibre_id'],'EPUB',
# index_is_id=True))))
## XXX 1==0 and
if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
custom_columns = self.gui.library_view.model().custom_columns
if ( collision in (CALIBREONLYSAVECOL) and
book['calibre_id'] and
prefs['savemetacol'] != '' and
prefs['savemetacol'] in custom_columns ):
label = custom_columns[prefs['savemetacol']]['label']
savedmetadata = db.get_custom(book['calibre_id'], label=label, index_is_id=True)
else:
savedmetadata = None
if savedmetadata:
adapter.setStoryMetadata(savedmetadata)
# let other exceptions percolate up.
story = adapter.getStoryMetadataOnly(get_cover=False)
else:
@@ -988,9 +998,9 @@ class FanFicFarePlugin(InterfaceAction):
label=field_metadata[k]['name']
key='calibre_std_'+k
if k == 'user_categories':
value=u', '.join(mi.get(k))
label=_('User Categories')
# if k == 'user_categories':
# value=u', '.join(mi.get(k))
# label=_('User Categories')
if label: # only if it has a human readable name.
book['calibre_columns'][key]={'val':value,'label':label}
@@ -998,15 +1008,16 @@ class FanFicFarePlugin(InterfaceAction):
# custom columns
for k, column in self.gui.library_view.model().custom_columns.iteritems():
key='calibre_cust_'+k[1:]
label=column['name']
value=db.get_custom(book['calibre_id'],
label=column['label'],
index_is_id=True)
# custom always have name.
book['calibre_columns'][key]={'val':value,'label':label}
logger.debug("%s(%s): %s"%(label,key,value))
if k != prefs['savemetacol']:
key='calibre_cust_'+k[1:]
label=column['name']
value=db.get_custom(book['calibre_id'],
label=column['label'],
index_is_id=True)
# custom always have name.
book['calibre_columns'][key]={'val':value,'label':label}
# logger.debug("%s(%s): %s"%(label,key,value))
################################################################################################################################################33
# set PI version instead of default.
@@ -1015,7 +1026,8 @@ class FanFicFarePlugin(InterfaceAction):
# all_metadata duplicates some data, but also includes extra_entries, etc.
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
book['publisher'] = story.getMetadata("site")
@@ -1042,7 +1054,7 @@ class FanFicFarePlugin(InterfaceAction):
book['timestamp'] = None # need *something* there for calibre.
if not merge:# skip all the collision code when d/ling for merging.
if collision in (CALIBREONLY):
if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
book['icon'] = 'metadata.png'
book['status'] = _('Meta')
@@ -1083,7 +1095,7 @@ class FanFicFarePlugin(InterfaceAction):
raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
## changed: add new book when CALIBREONLY if none found.
if collision == CALIBREONLY and not identicalbooks:
if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
collision = ADDNEW
options['collision'] = ADDNEW
@@ -1138,7 +1150,7 @@ class FanFicFarePlugin(InterfaceAction):
return
if book_id != None and collision != ADDNEW:
if collision in (CALIBREONLY):
if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
book['comment'] = _('Metadata collected.')
# don't need temp file created below.
return
@@ -1221,7 +1233,7 @@ class FanFicFarePlugin(InterfaceAction):
## No need to BG process when CALIBREONLY! Fake it.
#print("options:%s"%options)
if options['collision'] == CALIBREONLY:
if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
class NotJob(object):
def __init__(self,result):
self.failed=False
@@ -1306,10 +1318,10 @@ class FanFicFarePlugin(InterfaceAction):
logger.debug("add/update %s %s"%(book['title'],book['url']))
mi = self.make_mi_from_book(book)
if options['collision'] != CALIBREONLY:
if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
self.add_book_or_update_format(book,options,prefs,mi)
if options['collision'] == CALIBREONLY or \
if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
( (options['updatemeta'] or book['added']) and book['good'] ):
try:
self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1336,7 +1348,7 @@ class FanFicFarePlugin(InterfaceAction):
failed_list = filter(lambda x : not x['good'] , book_list)
failed_ids = [ x['calibre_id'] for x in failed_list ]
if options['collision'] != CALIBREONLY and \
if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
(prefs['addtolists'] or prefs['addtoreadlists']):
self.update_reading_lists(all_ids,add=True)
@@ -1400,7 +1412,7 @@ class FanFicFarePlugin(InterfaceAction):
if countpagesstats:
cp_plugin.count_statistics(all_ids,countpagesstats)
if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
@@ -1682,6 +1694,10 @@ class FanFicFarePlugin(InterfaceAction):
#print("all_metadata: %s"%book['all_metadata'])
custom_columns = self.gui.library_view.model().custom_columns
if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
label = custom_columns[prefs['savemetacol']]['label']
self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata()
#print("prefs['custom_cols'] %s"%prefs['custom_cols'])
for col, meta in prefs['custom_cols'].iteritems():
#print("setting %s to %s"%(col,meta))
calibre-plugin/jobs.py
+6 -2
View File
@@ -100,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
with fffbase:
from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
@@ -152,10 +152,11 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
outfile = book['outfile']
## No need to download at all. Shouldn't ever get down here.
if options['collision'] in (CALIBREONLY):
if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
book['comment'] = 'Metadata collected.'
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -175,6 +176,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -195,6 +197,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
if chaptercount == urlchaptercount:
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
return book
@@ -215,6 +218,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
book['savemetacol'] = story.dump_html_metadata()
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
calibre-plugin/prefs.py
+1
View File
@@ -98,6 +98,7 @@ default_prefs['countpagesstats'] = []
default_prefs['wordcountmissing'] = False
default_prefs['errorcol'] = ''
default_prefs['savemetacol'] = ''
default_prefs['custom_cols'] = {}
default_prefs['custom_cols_newonly'] = {}
default_prefs['allow_custcol_from_ini'] = True
fanficfare/adapters/base_adapter.py
+3 -3
View File
@@ -404,9 +404,9 @@ class BaseSiteAdapter(Configurable):
self.metadataDone = True
return self.story
def setStoryMetadata(self,metadatas):
if metadatas:
self.story.loads_metadata(metadatas)
def setStoryMetadata(self,metahtml):
if metahtml:
self.story.load_html_metadata(metahtml)
self.metadataDone = True
if not self.story.getMetadataRaw('dateUpdated'):
self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
fanficfare/configurable.py
+42 -1
View File
@@ -41,7 +41,38 @@ def re_compile(regex,line):
return re.compile(regex)
except Exception, e:
raise exceptions.RegularExpresssionFailed(e,regex,line)
# fall back labels.
titleLabels = {
'category':'Category',
'genre':'Genre',
'language':'Language',
'status':'Status',
'series':'Series',
'characters':'Characters',
'ships':'Relationships',
'datePublished':'Published',
'dateUpdated':'Updated',
'dateCreated':'Packaged',
'rating':'Rating',
'warnings':'Warnings',
'numChapters':'Chapters',
'numWords':'Words',
'site':'Site',
'storyId':'Story ID',
'authorId':'Author ID',
'extratags':'Extra Tags',
'title':'Title',
'storyUrl':'Story URL',
'description':'Summary',
'author':'Author',
'authorUrl':'Author URL',
'formatname':'File Format',
'formatext':'File Extension',
'siteabbrev':'Site Abbrev',
'version':'Downloader Version'
}
formatsections = ['html','txt','epub','mobi']
othersections = ['defaults','overrides']
@@ -607,3 +638,13 @@ class Configurable(object):
def get_config_list(self, sections, key):
return self.configuration.get_config_list(sections,key)
def get_label(self, entry):
if self.hasConfig(entry+"_label"):
label=self.getConfig(entry+"_label")
elif entry in titleLabels:
label=titleLabels[entry]
else:
label=entry.title()
return label
fanficfare/story.py
+55 -11
View File
@@ -26,6 +26,8 @@ import logging
logger = logging.getLogger(__name__)
import urlparse as up
import bs4
import exceptions
from htmlcleanup import conditionalRemoveEntities, removeAllEntities
from configurable import Configurable, re_compile
@@ -546,19 +548,61 @@ class Story(Configurable):
else:
return self.join_list(key,retlist)
# for saving a string-ified copy of metadata.
def dumps_metadata(self):
# md = {}
# for k,v in self.metadata.iteritems():
# if not k.startswith('calibre_'): # don't include items passed in for calibre cols.
# md[k]=v
# return json.dumps(md, default=datetime_encoder)
pass
# for saving an html-ified copy of metadata.
def dump_html_metadata(self):
lines=[]
for k,v in sorted(self.metadata.iteritems()):
classes=['metadata']
if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
classes.append("datetime")
val = v.isoformat()
elif isinstance(v,list):
classes.append("list")
val = "<ul>\n<li>%s</li>\n</ul>"%"</li>\n<li>".join(v)
elif isinstance(v, (int)):
classes.append("int")
val = v
else:
val = v
# for loading a string-ified copy of metadata.
def loads_metadata(self,s):
if not k.startswith('calibre_'): # don't include items passed in for calibre cols.
lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
self.get_label(k),
" ".join(classes),
k,val))
return "\n".join(lines)
# for loading an html-ified copy of metadata.
def load_html_metadata(self,data):
soup = bs4.BeautifulSoup(data,'html5lib')
for tag in soup.find_all('div','metadata'):
val = None
if 'datetime' in tag['class']:
v = tag.string
try:
val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
except ValueError:
try:
val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
except ValueError:
try:
val = datetime.datetime.strptime(v, '%Y-%m-%d')
except ValueError:
pass
elif 'list' in tag['class']:
val = []
for i in tag.find_all('li'):
val.append(i.string)
elif 'int' in tag['class']:
val = int(tag.string)
else:
val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
logger.debug("tag['id'](%s)=val(%s)"%(tag['id'],val))
if val:
self.metadata[tag['id']]=val
# self.metadata = json.loads(s, object_hook=datetime_decoder)
pass
def getMetadataRaw(self,key):
if self.isValidMetaEntry(key) and self.metadata.has_key(key):
fanficfare/writers/base_writer.py
+10 -39
View File
@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
self.adapter = adapter
self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
# fall back labels.
self.titleLabels = {
'category':'Category',
'genre':'Genre',
'language':'Language',
'status':'Status',
'series':'Series',
'characters':'Characters',
'ships':'Relationships',
'datePublished':'Published',
'dateUpdated':'Updated',
'dateCreated':'Packaged',
'rating':'Rating',
'warnings':'Warnings',
'numChapters':'Chapters',
'numWords':'Words',
'site':'Site',
'storyId':'Story ID',
'authorId':'Author ID',
'extratags':'Extra Tags',
'title':'Title',
'storyUrl':'Story URL',
'description':'Summary',
'author':'Author',
'authorUrl':'Author URL',
'formatname':'File Format',
'formatext':'File Extension',
'siteabbrev':'Site Abbrev',
'version':'Downloader Version'
}
self.story.setMetadata('formatname',self.getFormatName())
self.story.setMetadata('formatext',self.getFormatExt())
@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
TEMPLATE=WIDE_ENTRY
else:
TEMPLATE=ENTRY
if self.hasConfig(entry+"_label"):
label=self.getConfig(entry+"_label")
elif entry in self.titleLabels:
logger.debug("Using fallback label for %s_label"%entry)
label=self.titleLabels[entry]
else:
label="%s"%entry.title()
logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
label=self.get_label(entry)
# if self.hasConfig(entry+"_label"):
# label=self.getConfig(entry+"_label")
# elif entry in self.titleLabels:
# logger.debug("Using fallback label for %s_label"%entry)
# label=self.titleLabels[entry]
# else:
# label="%s"%entry.title()
# logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
# If the label for the title entry is empty, use the
# 'no title' option if there is one.
fanficfare/writers/writer_epub.py
+9 -8
View File
@@ -258,14 +258,15 @@ div { margin: 0pt; padding: 0pt; }
if self.isValidMetaEntry(entry):
val = self.story.getMetadata(entry)
if val and ( entry not in oldvalues or val != oldvalues[entry] ):
if self.hasConfig(entry+"_label"):
label=self.getConfig(entry+"_label")
elif entry in self.titleLabels:
logger.debug("Using fallback label for %s_label"%entry)
label=self.titleLabels[entry]
else:
label="%s"%entry.title()
logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
label=self.get_label(entry)
# if self.hasConfig(entry+"_label"):
# label=self.getConfig(entry+"_label")
# elif entry in self.titleLabels:
# logger.debug("Using fallback label for %s_label"%entry)
# label=self.titleLabels[entry]
# else:
# label="%s"%entry.title()
# logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
retval = retval + ENTRY.substitute({'id':entry,
'label':label,