diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index e1981bb..67285dd 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -10,6 +10,9 @@ __docformat__ = 'restructuredtext en' import traceback from functools import partial +import urllib +import email + from PyQt4 import QtGui from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout, QPushButton, QProgressDialog, QString, QLabel, @@ -30,6 +33,8 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \ import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog, ImageTitleLayout, get_icon) +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text + SKIP=u'Skip' ADDNEW=u'Add New Book' UPDATE=u'Update EPUB if New Chapters' @@ -73,6 +78,46 @@ class NotGoingToDownload(Exception): class DroppableQTextEdit(QTextEdit): def __init__(self,parent): QTextEdit.__init__(self,parent) + + def dropEvent(self,event): + # print("event:%s"%event) + # print("event.mimeData():%s"%event.mimeData()) + # print("event.mimeData().text():%s"%str(event.mimeData().text())) + # print("event.mimeData().data():%s"%str(event.mimeData().data())) + # print("event.mimeData().formats():%s"%[str(f) for f in event.mimeData().formats()]) + # for f in event.mimeData().formats(): + # try: + # print("event.mimeData().data('%s'):%s"%(f,event.mimeData().data(f))) + # except: + # print("failed %s"%f) + + mimetype='text/uri-list' + # print("event.mimeData().data('%s'):%s"%(mimetype,event.mimeData().data(mimetype))) + + urllist=[] + filelist="%s"%event.mimeData().data(mimetype) + for f in filelist.splitlines(): + #print("filename:%s"%f) + if f.endswith(".eml"): + fhandle = urllib.urlopen(f) + #print("file:\n%s\n\n"%fhandle.read()) + msg = email.message_from_file(fhandle) + if msg.is_multipart(): + for part in msg.walk(): + #print("part type:%s"%part.get_content_type()) + if part.get_content_type() == "text/html": + #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True))) + urllist.extend(get_urls_from_html(part.get_payload(decode=True))) + if part.get_content_type() == "text/plain": + #print("part content:text/plain") + # print("part content:%s"%part.get_payload(decode=True)) + urllist.extend(get_urls_from_text(part.get_payload(decode=True))) + else: + urllist.extend(get_urls_from_text("%s"%msg)) + + if urllist: + self.append("\n".join(urllist)) + return QTextEdit.dropEvent(self,event) def canInsertFromMimeData(self, source): if source.hasUrls(): diff --git a/fanficdownloader/geturls.py b/fanficdownloader/geturls.py index 93858a4..60cd132 100644 --- a/fanficdownloader/geturls.py +++ b/fanficdownloader/geturls.py @@ -27,8 +27,6 @@ from configurable import Configuration def get_urls_from_page(url,configuration=None): - normalized = set() # normalized url - retlist = [] # orig urls. if not configuration: configuration = Configuration("test1.com","EPUB") @@ -56,6 +54,16 @@ def get_urls_from_page(url,configuration=None): opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) data = opener.open(url).read() + return get_urls_from_html(data,url) + +def get_urls_from_html(data,url=None,configuration=None): + + normalized = set() # normalized url + retlist = [] # orig urls. + + if not configuration: + configuration = Configuration("test1.com","EPUB") + soup = BeautifulSoup(data) for a in soup.findAll('a'): @@ -81,6 +89,33 @@ def get_urls_from_page(url,configuration=None): return retlist +def get_urls_from_text(data,configuration=None): + + normalized = set() # normalized url + retlist = [] # orig urls. + + if not configuration: + configuration = Configuration("test1.com","EPUB") + + for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data): + # this (should) catch normal story links, some javascript + # 'are you old enough' links, and 'Report This' links. + # The 'normalized' set prevents duplicates. + if 'story.php' in href: + m = re.search(r"(?P(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href']) + if m != None: + href = form_url(None,m.group('sid')) + try: + href = href.replace('&index=1','') + adapter = adapters.getAdapter(configuration,href) + if adapter.story.getMetadata('storyUrl') not in normalized: + normalized.add(adapter.story.getMetadata('storyUrl')) + retlist.append(href) + except: + pass + + return retlist + def form_url(parenturl,url): url = url.strip() # ran across an image with a space in the # src. Browser handled it, so we'd better, too.