diff --git a/calibre-plugin/fff_plugin.py b/calibre-plugin/fff_plugin.py index 26503dc..cfab018 100644 --- a/calibre-plugin/fff_plugin.py +++ b/calibre-plugin/fff_plugin.py @@ -66,7 +66,8 @@ from calibre_plugins.fanficfare_plugin.fanficfare import ( adapters, exceptions) from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import ( - get_dcsource, get_dcsource_chaptercount, get_story_url_from_html) + get_dcsource, get_dcsource_chaptercount, get_story_url_from_html, + reset_orig_chapters_epub) from calibre_plugins.fanficfare_plugin.fanficfare.geturls import ( get_urls_from_page, get_urls_from_html,get_urls_from_text, @@ -335,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction): image='minusminus.png', triggered=partial(self.update_lists,add=False)) + self.menu.addSeparator() + self.get_list_action = self.create_menu_item_ex(self.menu, _('UnNew Selected books'), + unique_name='Get URLs from Selected Books', + image='rotate-right.png', + triggered=self.unnew_books) + self.menu.addSeparator() self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'), unique_name='Get URLs from Selected Books', @@ -415,6 +422,7 @@ class FanFicFarePlugin(InterfaceAction): return self.update_reading_lists(self.gui.library_view.get_selected_ids(),add) + self.unnew_books() def get_urls_from_imap_menu(self): @@ -555,6 +563,70 @@ class FanFicFarePlugin(InterfaceAction): show=True, show_copy_button=False) + def unnew_books(self): + '''Get list of URLs from existing books.''' + if not self.is_library_view(): + self.gui.status_bar.show_message(_('Can only UnNew books in library'), + 3000) + return + + if not self.gui.current_view().selectionModel().selectedRows() : + self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'), + 3000) + return + + book_list = map( partial(self.make_book_id_only), + self.gui.library_view.get_selected_ids() ) + + tdir = PersistentTemporaryDirectory(prefix='fanficfare_') + LoopProgressDialog(self.gui, + book_list, + partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir), + partial(self.get_unnew_books_finish, tdir=tdir), + init_label=_("UnNewing books..."), + win_title=_("UnNew Books"), + status_prefix=_("Books UnNewed")) + + def get_unnew_books_loop(self,book,db=None,tdir=None): + + if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True): + tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True) + + unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'], + suffix='.epub', + dir=tdir) + book['changed']=reset_orig_chapters_epub(tmp,unnewtmp) + if book['changed']: + db.add_format_with_hooks(book['calibre_id'], + 'EPUB', + unnewtmp, + index_is_id=True) + if prefs['deleteotherforms']: + fmts = db.formats(book['calibre_id'], index_is_id=True).split(',') + for fmt in fmts: + if fmt.lower() != formmapping['epub'].lower(): + logger.debug("deleteotherforms remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + elif prefs['autoconvert']: + ## 'Convert Book'.auto_convert_auto_add doesn't convert if + ## the format is already there. + fmt = calibre_prefs['output_format'] + # delete if there, but not if the format we just made. + if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True): + logger.debug("autoconvert remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + + def get_unnew_books_finish(self, book_list, tdir=None): + remove_dir(tdir) + if prefs['autoconvert']: + changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ] + if changed_ids: + self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000) + self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids) + def reject_list_urls(self): if self.is_library_view(): book_list = map( partial(self.make_book_id_only), diff --git a/fanficfare/epubutils.py b/fanficfare/epubutils.py index 6e51164..8f92ca2 100644 --- a/fanficfare/epubutils.py +++ b/fanficfare/epubutils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# -*- coding: utf-8 -*- from __future__ import (unicode_literals, division, absolute_import, print_function) @@ -11,7 +12,8 @@ import logging logger = logging.getLogger(__name__) import re, os, traceback -from zipfile import ZipFile +from collections import defaultdict +from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from xml.dom.minidom import parseString import bs4 as bs @@ -92,6 +94,7 @@ def get_update_data(inputio, soups = [] # list of xhmtl blocks urlsoups = {} # map of xhtml blocks by url images = {} # dict() longdesc->data + datamaps = defaultdict(dict) # map of data maps by url if getfilecount: # spin through the manifest--only place there are item tags. for item in contentdom.getElementsByTagName("item"): @@ -125,25 +128,45 @@ def get_update_data(inputio, logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc)) logger.warn("Exception: %s"%(unicode(e))) traceback.print_exc() - soup = soup.find('body') + bodysoup = soup.find('body') # ffdl epubs have chapter title h3 - h3 = soup.find('h3') + h3 = bodysoup.find('h3') if h3: h3.extract() # TtH epubs have chapter title h2 - h2 = soup.find('h2') + h2 = bodysoup.find('h2') if h2: h2.extract() - for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}): + for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}): skip.extract() - chapa = soup.find('a',{'class':'chapterurl'}) - if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. - urlsoups[chapa['href']] = soup - chapa.extract() + ## + #print("look for meta chapurl") + currenturl = None + chapurl = soup.find('meta',{'name':'chapterurl'}) + if chapurl: + if chapurl['content'] not in urlsoups: # keep first found if more than one. + #print("Found chapurl['content']:%s"%chapurl['content']) + currenturl = chapurl['content'] + urlsoups[chapurl['content']] = bodysoup + else: + # for older pre-meta. Only temp. + chapa = bodysoup.find('a',{'class':'chapterurl'}) + if chapa and chapa['href'] not in urlsoups: # keep first found if more than one. + urlsoups[chapa['href']] = bodysoup + currenturl = chapa['href'] + chapa.extract() + + chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'}) + if chapterorigtitle: + datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content'] - soups.append(soup) + chaptertitle = soup.find('meta',{'name':'chaptertitle'}) + if chaptertitle: + datamaps[currenturl]['chaptertitle'] = chaptertitle['content'] + + soups.append(bodysoup) filecount+=1 @@ -154,7 +177,8 @@ def get_update_data(inputio, #for k in images.keys(): #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups) + print("datamaps:%s"%datamaps) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps) def get_path_part(n): relpath = os.path.dirname(n) @@ -198,3 +222,61 @@ def get_story_url_from_html(inputio,_is_good_url=None): if _is_good_url == None or _is_good_url(ahref): return ahref return None + +def reset_orig_chapters_epub(inputio,outputio): + inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob + + ## Write mimetype file, must be first and uncompressed. + ## Older versions of python(2.4/5) don't allow you to specify + ## compression by individual file. + ## Overwrite if existing output file. + outputepub = ZipFile(outputio, 'w', compression=ZIP_STORED) + outputepub.debug = 3 + outputepub.writestr("mimetype", "application/epub+zip") + outputepub.close() + + ## Re-open file for content. + outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED) + outputepub.debug = 3 + + changed = False + + tocncx = inputepub.read('toc.ncx').decode('utf-8') + ## spin through file contents. + for zf in inputepub.namelist(): + if zf not in ['mimetype','toc.ncx'] : + data = inputepub.read(zf) + if isinstance(data,unicode): + print("\n\n\ndata is unicode\n\n\n") + if re.match(r'.*/file\d+\.xhtml',zf): + data = data.decode('utf-8') + soup = bs.BeautifulSoup(data,"html5lib") + + chapterorigtitle = None + tag = soup.find('meta',{'name':'chapterorigtitle'}) + if tag: + chapterorigtitle = tag['content'] + + chaptertitle = None + tag = soup.find('meta',{'name':'chaptertitle'}) + if tag: + chaptertitle = tag['content'] + + if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle: + origdata = data + origtocncx = tocncx + print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle)) + # changed = True + # data = data.replace(u'', + # u''+chapterorigtitle+u'') + data = data.replace(u'