From e637b4b073821d1c2ce776391938882b40fa8d66 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 3 Jul 2015 11:16:47 -0500 Subject: [PATCH] Save first version of UnNew feature for removing '(new)' marks on chapters. --- calibre-plugin/fff_plugin.py | 74 ++++++++++++++++++++- fanficfare/epubutils.py | 104 ++++++++++++++++++++++++++---- fanficfare/story.py | 21 +++--- fanficfare/writers/base_writer.py | 8 +-- fanficfare/writers/writer_epub.py | 26 +++++--- fanficfare/writers/writer_html.py | 10 +-- fanficfare/writers/writer_mobi.py | 10 +-- fanficfare/writers/writer_txt.py | 10 +-- 8 files changed, 214 insertions(+), 49 deletions(-) diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index 26503dc..cfab018 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -66,7 +66,8 @@ from calibre_plugins.fanficfare_plugin.fanficfare import ( adapters, exceptions) from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import ( - get_dcsource, get_dcsource_chaptercount, get_story_url_from_html) + get_dcsource, get_dcsource_chaptercount, get_story_url_from_html, + reset_orig_chapters_epub) from calibre_plugins.fanficfare_plugin.fanficfare.geturls import ( get_urls_from_page, get_urls_from_html,get_urls_from_text, @@ -335,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction): image='minusminus.png', triggered=partial(self.update_lists,add=False)) + self.menu.addSeparator() + self.get_list_action = self.create_menu_item_ex(self.menu, _('UnNew Selected books'), + unique_name='Get URLs from Selected Books', + image='rotate-right.png', + triggered=self.unnew_books) + self.menu.addSeparator() self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'), unique_name='Get URLs from Selected Books', @@ -415,6 +422,7 @@ class FanFicFarePlugin(InterfaceAction): return self.update_reading_lists(self.gui.library_view.get_selected_ids(),add) + self.unnew_books() def get_urls_from_imap_menu(self): @@ -555,6 +563,70 @@ class FanFicFarePlugin(InterfaceAction): show=True, show_copy_button=False) + def unnew_books(self): + '''Get list of URLs from existing books.''' + if not self.is_library_view(): + self.gui.status_bar.show_message(_('Can only UnNew books in library'), + 3000) + return + + if not self.gui.current_view().selectionModel().selectedRows() : + self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'), + 3000) + return + + book_list = map( partial(self.make_book_id_only), + self.gui.library_view.get_selected_ids() ) + + tdir = PersistentTemporaryDirectory(prefix='fanficfare_') + LoopProgressDialog(self.gui, + book_list, + partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir), + partial(self.get_unnew_books_finish, tdir=tdir), + init_label=_("UnNewing books..."), + win_title=_("UnNew Books"), + status_prefix=_("Books UnNewed")) + + def get_unnew_books_loop(self,book,db=None,tdir=None): + + if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True): + tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True) + + unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + book['changed']=reset_orig_chapters_epub(tmp,unnewtmp) + if book['changed']: + db.add_format_with_hooks(book['calibre_id'], + 'EPUB', + unnewtmp, + index_is_id=True) + if prefs['deleteotherforms']: + fmts = db.formats(book['calibre_id'], index_is_id=True).split(',') + for fmt in fmts: + if fmt.lower() != formmapping['epub'].lower(): + logger.debug("deleteotherforms remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + elif prefs['autoconvert']: + ## 'Convert Book'.auto_convert_auto_add doesn't convert if + ## the format is already there. + fmt = calibre_prefs['output_format'] + # delete if there, but not if the format we just made. + if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True): + logger.debug("autoconvert remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + + def get_unnew_books_finish(self, book_list, tdir=None): + remove_dir(tdir) + if prefs['autoconvert']: + changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ] + if changed_ids: + self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000) + self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids) + def reject_list_urls(self): if self.is_library_view(): book_list = map( partial(self.make_book_id_only), diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 6e51164..8f92ca2 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) @@ -11,7 +12,8 @@ import logging logger = logging.getLogger(__name__) import re, os, traceback -from zipfile import ZipFile +from collections import defaultdict +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from xml.dom.minidom import parseString import bs4 as bs @@ -92,6 +94,7 @@ def get_update_data(inputio, soups = [] # list of xhmtl blocks urlsoups = {} # map of xhtml blocks by url images = {} # dict() longdesc->data + datamaps = defaultdict(dict) # map of data maps by url if getfilecount: # spin through the manifest--only place there are item tags. for item in contentdom.getElementsByTagName("item"): @@ -125,25 +128,45 @@ def get_update_data(inputio, logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc)) logger.warn("Exception: %s"%(unicode(e))) traceback.print_exc() - soup = soup.find('body') + bodysoup = soup.find('body') # ffdl epubs have chapter title h3 - h3 = soup.find('h3') + h3 = bodysoup.find('h3') if h3: h3.extract() # TtH epubs have chapter title h2 - h2 = soup.find('h2') + h2 = bodysoup.find('h2') if h2: h2.extract() - for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}): + for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}): skip.extract() - chapa = soup.find('a',{'class':'chapterurl'}) - if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. - urlsoups[chapa['href']] = soup - chapa.extract() + ## + #print("look for meta chapurl") + currenturl = None + chapurl = soup.find('meta',{'name':'chapterurl'}) + if chapurl: + if chapurl['content'] not in urlsoups: # keep first found if more than one. + #print("Found chapurl['content']:%s"%chapurl['content']) + currenturl = chapurl['content'] + urlsoups[chapurl['content']] = bodysoup + else: + # for older pre-meta. Only temp. + chapa = bodysoup.find('a',{'class':'chapterurl'}) + if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. + urlsoups[chapa['href']] = bodysoup + currenturl = chapa['href'] + chapa.extract() + + chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'}) + if chapterorigtitle: + datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content'] - soups.append(soup) + chaptertitle = soup.find('meta',{'name':'chaptertitle'}) + if chaptertitle: + datamaps[currenturl]['chaptertitle'] = chaptertitle['content'] + + soups.append(bodysoup) filecount+=1 @@ -154,7 +177,8 @@ def get_update_data(inputio, #for k in images.keys(): #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups) + print("datamaps:%s"%datamaps) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps) def get_path_part(n): relpath = os.path.dirname(n) @@ -198,3 +222,61 @@ def get_story_url_from_html(inputio,_is_good_url=None): if _is_good_url == None or _is_good_url(ahref): return ahref return None + +def reset_orig_chapters_epub(inputio,outputio): + inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob + + ## Write mimetype file, must be first and uncompressed. + ## Older versions of python(2.4/5) don't allow you to specify + ## compression by individual file. + ## Overwrite if existing output file. + outputepub = ZipFile(outputio, 'w', compression=ZIP_STORED) + outputepub.debug = 3 + outputepub.writestr("mimetype", "application/epub+zip") + outputepub.close() + + ## Re-open file for content. + outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED) + outputepub.debug = 3 + + changed = False + + tocncx = inputepub.read('toc.ncx').decode('utf-8') + ## spin through file contents. + for zf in inputepub.namelist(): + if zf not in ['mimetype','toc.ncx'] : + data = inputepub.read(zf) + if isinstance(data,unicode): + print("\n\n\ndata is unicode\n\n\n") + if re.match(r'.*/file\d+\.xhtml',zf): + data = data.decode('utf-8') + soup = bs.BeautifulSoup(data,"html5lib") + + chapterorigtitle = None + tag = soup.find('meta',{'name':'chapterorigtitle'}) + if tag: + chapterorigtitle = tag['content'] + + chaptertitle = None + tag = soup.find('meta',{'name':'chaptertitle'}) + if tag: + chaptertitle = tag['content'] + + if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle: + origdata = data + origtocncx = tocncx + print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) + # changed = True + # data = data.replace(u'', + # u''+chapterorigtitle+u'') + data = data.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') + data = data.replace(u'

'+chaptertitle+u'

',u'

'+chapterorigtitle+u'

') + tocncx = tocncx.replace(u''+chaptertitle+u'',u''+chapterorigtitle+u'') + changed = ( origdata != data or origtocncx != tocncx ) + outputepub.writestr(zf,data.encode('utf-8')) + else: + outputepub.writestr(zf,data) + + outputepub.writestr('toc.ncx',tocncx.encode('utf-8')) + + return changed diff --git a/fanficfare/story.py b/fanficfare/story.py index dd578ce..017a469 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -16,6 +16,7 @@ # import os, re +from collections import namedtuple import urlparse import string import json @@ -32,6 +33,8 @@ import exceptions from htmlcleanup import conditionalRemoveEntities, removeAllEntities from configurable import Configurable, re_compile +Chapter = namedtuple('Chapter', 'url title html origtitle') + SPACE_REPLACE=u'\s' SPLIT_META=u'\,' @@ -412,7 +415,7 @@ class Story(Configurable): except: self.metadata = {'version':'4.4'} self.in_ex_cludes = {} - self.chapters = [] # chapters will be tuples of (url,title,html) + self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc) self.imgurls = [] self.imgtuples = [] @@ -844,22 +847,24 @@ class Story(Configurable): if self.getConfig('strip_chapter_numbers') and \ self.getConfig('chapter_title_strip_pattern'): title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title) + newtitle=title if marknewchap: - title=u'(new) %s'%title - self.chapters.append( (url,title,html) ) + newtitle=u'(new) %s'%title + self.chapters.append( Chapter(url,newtitle,html,title) ) def getChapters(self,fortoc=False): - "Chapters will be tuples of (title,html)" + "Chapters will be Chapter namedtuples of (url,title,html,new)" retval = [] ## only add numbers if more than one chapter. if len(self.chapters) > 1 and \ (self.getConfig('add_chapter_numbers') == "true" \ or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \ and self.getConfig('chapter_title_add_pattern'): - for index, (url,title,html) in enumerate(self.chapters): - retval.append( (url, - string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}), - html) ) + for index, chap in enumerate(self.chapters): + retval.append( Chapter(chap.url, + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.title}), + chap.html, + string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':chap.origtitle})) ) else: retval = self.chapters diff --git a/fanficfare/writers/base_writer.py b/fanficfare/writers/base_writer.py index 3fdde1c..df5885c 100644 --- a/fanficfare/writers/base_writer.py +++ b/fanficfare/writers/base_writer.py @@ -148,12 +148,12 @@ class BaseStoryWriter(Configurable): self._write(out,START.substitute(self.story.getAllMetadata())) - for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): - if html: - self._write(out,ENTRY.substitute({'chapter':title, + for index, chap in enumerate(self.story.getChapters(fortoc=True)): + if chap.html: + self._write(out,ENTRY.substitute({'chapter':chap.title, 'number':index+1, 'index':"%04d"%(index+1), - 'url':url})) + 'url':chap.url})) self._write(out,END.substitute(self.story.getAllMetadata())) diff --git a/fanficfare/writers/writer_epub.py b/fanficfare/writers/writer_epub.py index d59832b..3e0fcfb 100644 --- a/fanficfare/writers/writer_epub.py +++ b/fanficfare/writers/writer_epub.py @@ -28,7 +28,7 @@ import re from xml.dom.minidom import parse, parseString, getDOMImplementation from base_writer import * -from ..htmlcleanup import stripHTML +from ..htmlcleanup import stripHTML,removeEntities logger = logging.getLogger(__name__) @@ -133,6 +133,9 @@ ${value}
${chapter} + + +

${chapter}

@@ -502,13 +505,13 @@ div { margin: 0pt; padding: 0pt; } items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log")) itemrefs.append("log_page") - for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)): - if html: + for index, chap in enumerate(self.story.getChapters(fortoc=True)): + if chap.html: i=index+1 items.append(("file%04d"%i, "OEBPS/file%04d.xhtml"%i, "application/xhtml+xml", - title)) + chap.title)) itemrefs.append("file%04d"%i) manifest = contentdom.createElement("manifest") @@ -650,13 +653,16 @@ div { margin: 0pt; padding: 0pt; } else: CHAPTER_END = self.EPUB_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logger.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): # (url,title,html) + if chap.html: + logger.debug('Writing chapter text for: %s' % chap.title) + vals={'url':removeEntities(chap.url), + 'chapter':chap.title, + 'origchapter':chap.origtitle, + 'index':"%04d"%(index+1), + 'number':index+1} fullhtml = CHAPTER_START.substitute(vals) + \ - '' + \ - html + CHAPTER_END.substitute(vals) + chap.html + CHAPTER_END.substitute(vals) # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big diff --git a/fanficfare/writers/writer_html.py b/fanficfare/writers/writer_html.py index 03f151e..c99e396 100644 --- a/fanficfare/writers/writer_html.py +++ b/fanficfare/writers/writer_html.py @@ -128,12 +128,12 @@ ${output_css} else: CHAPTER_END = self.HTML_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logging.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logging.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} self._write(out,CHAPTER_START.substitute(vals)) - self._write(out,html) + self._write(out,chap.html) self._write(out,CHAPTER_END.substitute(vals)) self._write(out,FILE_END.substitute(self.story.getAllMetadata())) diff --git a/fanficfare/writers/writer_mobi.py b/fanficfare/writers/writer_mobi.py index 1018b56..ca8ed5c 100644 --- a/fanficfare/writers/writer_mobi.py +++ b/fanficfare/writers/writer_mobi.py @@ -161,11 +161,11 @@ ${value}
else: CHAPTER_END = self.MOBI_CHAPTER_END - for index, (url,title,html) in enumerate(self.story.getChapters()): - if html: - logger.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} - fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals) + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logger.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} + fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals) # ffnet(& maybe others) gives the whole chapter text # as one line. This causes problems for nook(at # least) when the chapter size starts getting big diff --git a/fanficfare/writers/writer_txt.py b/fanficfare/writers/writer_txt.py index c91726d..91b3ac5 100644 --- a/fanficfare/writers/writer_txt.py +++ b/fanficfare/writers/writer_txt.py @@ -154,12 +154,12 @@ End file. else: CHAPTER_END = self.TEXT_CHAPTER_END - for index, (url, title,html) in enumerate(self.story.getChapters()): - if html: - logging.debug('Writing chapter text for: %s' % title) - vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1} + for index, chap in enumerate(self.story.getChapters()): + if chap.html: + logging.debug('Writing chapter text for: %s' % chap.title) + vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1} self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals))))) - self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width))) + self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width))) self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals))))) self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))