diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index f90d99b..746a87f 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -323,6 +323,9 @@ class ConfigWidget(QWidget): # error column prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()))) + # metadata column + prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex()))) + # cust cols tab colsmap = {} for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems(): @@ -680,14 +683,15 @@ class PersonalIniTab(QWidget): self.personalini = d.get_plain_text() def show_showcalcols(self): - lines=[('calibre_std_user_categories',_('User Categories'))] + lines=[]#[('calibre_std_user_categories',_('User Categories'))] for k,f in field_metadata.iteritems(): if f['name']: # only if it has a human readable name. lines.append(('calibre_std_'+k,f['name'])) for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems(): - # custom always have name. - lines.append(('calibre_cust_'+k[1:],column['name'])) + if k != prefs['savemetacol']: + # custom always have name. + lines.append(('calibre_cust_'+k[1:],column['name'])) lines.sort() # sort by key. @@ -1235,6 +1239,21 @@ class CustomColumnsTab(QWidget): self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol'])) horz.addWidget(self.errorcol) self.l.addLayout(horz) + + horz = QHBoxLayout() + label = QLabel(_("Saved Metadata Column:")) + tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.
The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.
(Long Text columns only.)") + label.setToolTip(tooltip) + horz.addWidget(label) + self.savemetacol = QComboBox(self) + self.savemetacol.setToolTip(tooltip) + self.savemetacol.addItem('','none') + for key, column in custom_columns.iteritems(): + if column['datatype'] in ('comments'): + self.savemetacol.addItem(column['name'],key) + self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol'])) + horz.addWidget(self.savemetacol) + self.l.addLayout(horz) #print("prefs['custom_cols'] %s"%prefs['custom_cols']) diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 4943105..38fc19d 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters') UPDATEALWAYS=_('Update EPUB Always') OVERWRITE=_('Overwrite if Newer') OVERWRITEALWAYS=_('Overwrite Always') -CALIBREONLY=_('Update Calibre Metadata Only') +CALIBREONLY=_('Update Calibre Metadata from Web Site') +CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column') collision_order=[SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS, - CALIBREONLY,] + CALIBREONLY, + CALIBREONLYSAVECOL,] # best idea I've had for how to deal with config/pref saving the # collision name in english. @@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always' SAVE_OVERWRITE='Overwrite if Newer' SAVE_OVERWRITEALWAYS='Overwrite Always' SAVE_CALIBREONLY='Update Calibre Metadata Only' +SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)' save_collisions={ SKIP:SAVE_SKIP, ADDNEW:SAVE_ADDNEW, @@ -112,6 +115,7 @@ save_collisions={ SAVE_OVERWRITE:OVERWRITE, SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS, SAVE_CALIBREONLY:CALIBREONLY, + SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL, } anthology_collision_order=[UPDATE, diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index c0227f0..399cf61 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -86,6 +86,7 @@ from calibre_plugins.fanficfare_plugin.dialogs import ( LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog, OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, + CALIBREONLYSAVECOL, NotGoingToDownload, RejectUrlEntry ) # because calibre immediately transforms html into zip and don't want @@ -902,12 +903,21 @@ class FanFicFarePlugin(InterfaceAction): options['cookiejar'] = adapter.get_empty_cookiejar() adapter.set_cookiejar(options['cookiejar']) - ## XXX get_epub_metadata works, but how to use it? - if 1==0 and collision in (CALIBREONLY) and \ - fileform == 'epub' and \ - db.has_format(book['calibre_id'],'EPUB',index_is_id=True): - # adapter.setStoryMetadata(get_epub_metadatas(StringIO(db.format(book['calibre_id'],'EPUB', - # index_is_id=True)))) + ## XXX 1==0 and + if collision in (CALIBREONLY, CALIBREONLYSAVECOL): + custom_columns = self.gui.library_view.model().custom_columns + if ( collision in (CALIBREONLYSAVECOL) and + book['calibre_id'] and + prefs['savemetacol'] != '' and + prefs['savemetacol'] in custom_columns ): + label = custom_columns[prefs['savemetacol']]['label'] + savedmetadata = db.get_custom(book['calibre_id'], label=label, index_is_id=True) + else: + savedmetadata = None + + if savedmetadata: + adapter.setStoryMetadata(savedmetadata) + # let other exceptions percolate up. story = adapter.getStoryMetadataOnly(get_cover=False) else: @@ -988,9 +998,9 @@ class FanFicFarePlugin(InterfaceAction): label=field_metadata[k]['name'] key='calibre_std_'+k - if k == 'user_categories': - value=u', '.join(mi.get(k)) - label=_('User Categories') + # if k == 'user_categories': + # value=u', '.join(mi.get(k)) + # label=_('User Categories') if label: # only if it has a human readable name. book['calibre_columns'][key]={'val':value,'label':label} @@ -998,15 +1008,16 @@ class FanFicFarePlugin(InterfaceAction): # custom columns for k, column in self.gui.library_view.model().custom_columns.iteritems(): - key='calibre_cust_'+k[1:] - label=column['name'] - value=db.get_custom(book['calibre_id'], - label=column['label'], - index_is_id=True) - # custom always have name. - book['calibre_columns'][key]={'val':value,'label':label} - logger.debug("%s(%s): %s"%(label,key,value)) - + if k != prefs['savemetacol']: + key='calibre_cust_'+k[1:] + label=column['name'] + value=db.get_custom(book['calibre_id'], + label=column['label'], + index_is_id=True) + # custom always have name. + book['calibre_columns'][key]={'val':value,'label':label} + # logger.debug("%s(%s): %s"%(label,key,value)) + ################################################################################################################################################33 # set PI version instead of default. @@ -1015,7 +1026,8 @@ class FanFicFarePlugin(InterfaceAction): # all_metadata duplicates some data, but also includes extra_entries, etc. book['all_metadata'] = story.getAllMetadata(removeallentities=True) - + book['savemetacol'] = story.dump_html_metadata() + book['title'] = story.getMetadata("title", removeallentities=True) book['author_sort'] = book['author'] = story.getList("author", removeallentities=True) book['publisher'] = story.getMetadata("site") @@ -1042,7 +1054,7 @@ class FanFicFarePlugin(InterfaceAction): book['timestamp'] = None # need *something* there for calibre. if not merge:# skip all the collision code when d/ling for merging. - if collision in (CALIBREONLY): + if collision in (CALIBREONLY, CALIBREONLYSAVECOL): book['icon'] = 'metadata.png' book['status'] = _('Meta') @@ -1083,7 +1095,7 @@ class FanFicFarePlugin(InterfaceAction): raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png") ## changed: add new book when CALIBREONLY if none found. - if collision == CALIBREONLY and not identicalbooks: + if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks: collision = ADDNEW options['collision'] = ADDNEW @@ -1138,7 +1150,7 @@ class FanFicFarePlugin(InterfaceAction): return if book_id != None and collision != ADDNEW: - if collision in (CALIBREONLY): + if collision in (CALIBREONLY, CALIBREONLYSAVECOL): book['comment'] = _('Metadata collected.') # don't need temp file created below. return @@ -1221,7 +1233,7 @@ class FanFicFarePlugin(InterfaceAction): ## No need to BG process when CALIBREONLY! Fake it. #print("options:%s"%options) - if options['collision'] == CALIBREONLY: + if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL): class NotJob(object): def __init__(self,result): self.failed=False @@ -1306,10 +1318,10 @@ class FanFicFarePlugin(InterfaceAction): logger.debug("add/update %s %s"%(book['title'],book['url'])) mi = self.make_mi_from_book(book) - if options['collision'] != CALIBREONLY: + if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL): self.add_book_or_update_format(book,options,prefs,mi) - if options['collision'] == CALIBREONLY or \ + if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \ ( (options['updatemeta'] or book['added']) and book['good'] ): try: self.update_metadata(db, book['calibre_id'], book, mi, options) @@ -1336,7 +1348,7 @@ class FanFicFarePlugin(InterfaceAction): failed_list = filter(lambda x : not x['good'] , book_list) failed_ids = [ x['calibre_id'] for x in failed_list ] - if options['collision'] != CALIBREONLY and \ + if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \ (prefs['addtolists'] or prefs['addtoreadlists']): self.update_reading_lists(all_ids,add=True) @@ -1400,7 +1412,7 @@ class FanFicFarePlugin(InterfaceAction): if countpagesstats: cp_plugin.count_statistics(all_ids,countpagesstats) - if prefs['autoconvert'] and options['collision'] != CALIBREONLY: + if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL): self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000) self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids) @@ -1682,6 +1694,10 @@ class FanFicFarePlugin(InterfaceAction): #print("all_metadata: %s"%book['all_metadata']) custom_columns = self.gui.library_view.model().custom_columns + if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns: + label = custom_columns[prefs['savemetacol']]['label'] + self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata() + #print("prefs['custom_cols'] %s"%prefs['custom_cols']) for col, meta in prefs['custom_cols'].iteritems(): #print("setting %s to %s"%(col,meta)) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index f2b411e..dc667f7 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -100,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): with fffbase: from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload, - OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY) + OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL) from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data @@ -152,10 +152,11 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): outfile = book['outfile'] ## No need to download at all. Shouldn't ever get down here. - if options['collision'] in (CALIBREONLY): + if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL): logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...") book['comment'] = 'Metadata collected.' book['all_metadata'] = story.getAllMetadata(removeallentities=True) + book['savemetacol'] = story.dump_html_metadata() ## checks were done earlier, it's new or not dup or newer--just write it. elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \ @@ -175,6 +176,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): writer.writeStory(outfilename=outfile, forceOverwrite=True) book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters")) book['all_metadata'] = story.getAllMetadata(removeallentities=True) + book['savemetacol'] = story.dump_html_metadata() ## checks were done earlier, just update it. elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS): @@ -195,6 +197,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): if chaptercount == urlchaptercount: book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount book['all_metadata'] = story.getAllMetadata(removeallentities=True) + book['savemetacol'] = story.dump_html_metadata() book['outfile'] = book['epub_for_update'] # for anthology merge ops. return book @@ -215,6 +218,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x): book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\ (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount) book['all_metadata'] = story.getAllMetadata(removeallentities=True) + book['savemetacol'] = story.dump_html_metadata() if options['smarten_punctuation'] and options['fileform'] == "epub" \ and calibre_version >= (0, 9, 39): diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py index 302d560..f751577 100644 --- a/calibre-plugin/prefs.py +++ b/calibre-plugin/prefs.py @@ -98,6 +98,7 @@ default_prefs['countpagesstats'] = [] default_prefs['wordcountmissing'] = False default_prefs['errorcol'] = '' +default_prefs['savemetacol'] = '' default_prefs['custom_cols'] = {} default_prefs['custom_cols_newonly'] = {} default_prefs['allow_custcol_from_ini'] = True diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 9968ce6..2b76d69 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -404,9 +404,9 @@ class BaseSiteAdapter(Configurable): self.metadataDone = True return self.story - def setStoryMetadata(self,metadatas): - if metadatas: - self.story.loads_metadata(metadatas) + def setStoryMetadata(self,metahtml): + if metahtml: + self.story.load_html_metadata(metahtml) self.metadataDone = True if not self.story.getMetadataRaw('dateUpdated'): self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished')) diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index 69057ea..9d4623c 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -41,7 +41,38 @@ def re_compile(regex,line): return re.compile(regex) except Exception, e: raise exceptions.RegularExpresssionFailed(e,regex,line) - + +# fall back labels. +titleLabels = { + 'category':'Category', + 'genre':'Genre', + 'language':'Language', + 'status':'Status', + 'series':'Series', + 'characters':'Characters', + 'ships':'Relationships', + 'datePublished':'Published', + 'dateUpdated':'Updated', + 'dateCreated':'Packaged', + 'rating':'Rating', + 'warnings':'Warnings', + 'numChapters':'Chapters', + 'numWords':'Words', + 'site':'Site', + 'storyId':'Story ID', + 'authorId':'Author ID', + 'extratags':'Extra Tags', + 'title':'Title', + 'storyUrl':'Story URL', + 'description':'Summary', + 'author':'Author', + 'authorUrl':'Author URL', + 'formatname':'File Format', + 'formatext':'File Extension', + 'siteabbrev':'Site Abbrev', + 'version':'Downloader Version' + } + formatsections = ['html','txt','epub','mobi'] othersections = ['defaults','overrides'] @@ -607,3 +638,13 @@ class Configurable(object): def get_config_list(self, sections, key): return self.configuration.get_config_list(sections,key) + + def get_label(self, entry): + if self.hasConfig(entry+"_label"): + label=self.getConfig(entry+"_label") + elif entry in titleLabels: + label=titleLabels[entry] + else: + label=entry.title() + return label + diff --git a/fanficfare/story.py b/fanficfare/story.py index d241092..0df52b6 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -26,6 +26,8 @@ import logging logger = logging.getLogger(__name__) import urlparse as up +import bs4 + import exceptions from htmlcleanup import conditionalRemoveEntities, removeAllEntities from configurable import Configurable, re_compile @@ -546,19 +548,61 @@ class Story(Configurable): else: return self.join_list(key,retlist) - # for saving a string-ified copy of metadata. - def dumps_metadata(self): - # md = {} - # for k,v in self.metadata.iteritems(): - # if not k.startswith('calibre_'): # don't include items passed in for calibre cols. - # md[k]=v - # return json.dumps(md, default=datetime_encoder) - pass + # for saving an html-ified copy of metadata. + def dump_html_metadata(self): + lines=[] + for k,v in sorted(self.metadata.iteritems()): + classes=['metadata'] + if isinstance(v, (datetime.date, datetime.datetime, datetime.time)): + classes.append("datetime") + val = v.isoformat() + elif isinstance(v,list): + classes.append("list") + val = ""%"\n
  • ".join(v) + elif isinstance(v, (int)): + classes.append("int") + val = v + else: + val = v - # for loading a string-ified copy of metadata. - def loads_metadata(self,s): + if not k.startswith('calibre_'): # don't include items passed in for calibre cols. + lines.append("

    %s:

    %s

    \n"%( + self.get_label(k), + " ".join(classes), + k,val)) + return "\n".join(lines) + + # for loading an html-ified copy of metadata. + def load_html_metadata(self,data): + soup = bs4.BeautifulSoup(data,'html5lib') + for tag in soup.find_all('div','metadata'): + val = None + if 'datetime' in tag['class']: + v = tag.string + try: + val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f') + except ValueError: + try: + val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S') + except ValueError: + try: + val = datetime.datetime.strptime(v, '%Y-%m-%d') + except ValueError: + pass + elif 'list' in tag['class']: + val = [] + for i in tag.find_all('li'): + val.append(i.string) + elif 'int' in tag['class']: + val = int(tag.string) + else: + val = unicode("\n".join([ unicode(c) for c in tag.contents ])) + + logger.debug("tag['id'](%s)=val(%s)"%(tag['id'],val)) + if val: + self.metadata[tag['id']]=val + # self.metadata = json.loads(s, object_hook=datetime_decoder) - pass def getMetadataRaw(self,key): if self.isValidMetaEntry(key) and self.metadata.has_key(key): diff --git a/fanficfare/writers/base_writer.py b/fanficfare/writers/base_writer.py index f049c4c..3fdde1c 100644 --- a/fanficfare/writers/base_writer.py +++ b/fanficfare/writers/base_writer.py @@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable): self.adapter = adapter self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially. - # fall back labels. - self.titleLabels = { - 'category':'Category', - 'genre':'Genre', - 'language':'Language', - 'status':'Status', - 'series':'Series', - 'characters':'Characters', - 'ships':'Relationships', - 'datePublished':'Published', - 'dateUpdated':'Updated', - 'dateCreated':'Packaged', - 'rating':'Rating', - 'warnings':'Warnings', - 'numChapters':'Chapters', - 'numWords':'Words', - 'site':'Site', - 'storyId':'Story ID', - 'authorId':'Author ID', - 'extratags':'Extra Tags', - 'title':'Title', - 'storyUrl':'Story URL', - 'description':'Summary', - 'author':'Author', - 'authorUrl':'Author URL', - 'formatname':'File Format', - 'formatext':'File Extension', - 'siteabbrev':'Site Abbrev', - 'version':'Downloader Version' - } self.story.setMetadata('formatname',self.getFormatName()) self.story.setMetadata('formatext',self.getFormatExt()) @@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable): TEMPLATE=WIDE_ENTRY else: TEMPLATE=ENTRY - - if self.hasConfig(entry+"_label"): - label=self.getConfig(entry+"_label") - elif entry in self.titleLabels: - logger.debug("Using fallback label for %s_label"%entry) - label=self.titleLabels[entry] - else: - label="%s"%entry.title() - logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) + + label=self.get_label(entry) + # if self.hasConfig(entry+"_label"): + # label=self.getConfig(entry+"_label") + # elif entry in self.titleLabels: + # logger.debug("Using fallback label for %s_label"%entry) + # label=self.titleLabels[entry] + # else: + # label="%s"%entry.title() + # logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) # If the label for the title entry is empty, use the # 'no title' option if there is one. diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index 07a9654..abbb646 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -258,14 +258,15 @@ div { margin: 0pt; padding: 0pt; } if self.isValidMetaEntry(entry): val = self.story.getMetadata(entry) if val and ( entry not in oldvalues or val != oldvalues[entry] ): - if self.hasConfig(entry+"_label"): - label=self.getConfig(entry+"_label") - elif entry in self.titleLabels: - logger.debug("Using fallback label for %s_label"%entry) - label=self.titleLabels[entry] - else: - label="%s"%entry.title() - logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) + label=self.get_label(entry) + # if self.hasConfig(entry+"_label"): + # label=self.getConfig(entry+"_label") + # elif entry in self.titleLabels: + # logger.debug("Using fallback label for %s_label"%entry) + # label=self.titleLabels[entry] + # else: + # label="%s"%entry.title() + # logger.debug("No known label for %s, fallback to '%s'"%(entry,label)) retval = retval + ENTRY.substitute({'id':entry, 'label':label,