From 83ea401eaf38bf6736f375c7cb9c54bde40e4725 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 12 Feb 2015 11:41:23 -0600 Subject: [PATCH] Save IMAP story url fetch feature. Needs translation strings updated. --- calibre-plugin/config.py | 103 +++++++++++++++++++++++++++++++--- calibre-plugin/dialogs.py | 48 +++++++++++++++- calibre-plugin/ffdl_plugin.py | 55 +++++++++++++++++- calibre-plugin/prefs.py | 9 ++- fanficdownloader/geturls.py | 57 ++++++++++++++++++- 5 files changed, 259 insertions(+), 13 deletions(-) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 04fdb81..17c6102 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) __license__ = 'GPL v3' -__copyright__ = '2014, Jim Miller' +__copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' import logging @@ -14,15 +14,15 @@ import traceback, copy, threading from collections import OrderedDict try: - from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, - QLineEdit, QFont, QWidget, QTextEdit, QComboBox, + from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, + QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QScrollArea, - QDialogButtonBox, QGroupBox ) + QDialogButtonBox, QGroupBox, Qt ) except ImportError as e: - from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, - QLineEdit, QFont, QWidget, QTextEdit, QComboBox, + from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, + QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QScrollArea, - QDialogButtonBox, QGroupBox ) + QDialogButtonBox, QGroupBox, Qt ) try: from calibre.gui2 import QVariant del QVariant @@ -216,6 +216,9 @@ class ConfigWidget(QWidget): self.cust_columns_tab = CustomColumnsTab(self, plugin_action) tab_widget.addTab(self.cust_columns_tab, _('Custom Columns')) + self.imap_tab = ImapTab(self, plugin_action) + tab_widget.addTab(self.imap_tab, _('Email Settings')) + self.other_tab = OtherTab(self, plugin_action) tab_widget.addTab(self.other_tab, _('Other')) @@ -319,6 +322,13 @@ class ConfigWidget(QWidget): prefs['custom_cols_newonly'] = colsnewonly prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked() + + prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()) + prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()) + prefs['imappass'] = unicode(self.imap_tab.imappass.text()) + prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()) + prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked() + prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked() prefs.save_to_db() @@ -1118,3 +1128,82 @@ class StandardColumnsTab(QWidget): self.l.insertStretch(-1) +class ImapTab(QWidget): + + def __init__(self, parent_dialog, plugin_action): + self.parent_dialog = parent_dialog + self.plugin_action = plugin_action + QWidget.__init__(self) + + self.l = QGridLayout() + self.setLayout(self.l) + row=0 + + label = QLabel(_('These settings will allow FFDL to fetch story URLs from your email account. It will only look for story URLs in unread emails in the folder specified below.')) + label.setWordWrap(True) + self.l.addWidget(label,row,0,1,-1) + row+=1 + + label = QLabel(_('IMAP Server Name')) + tooltip = _("Name of IMAP server--must allow IMAP4 with SSL. Eg: imap.gmail.com") + label.setToolTip(tooltip) + self.l.addWidget(label,row,0) + self.imapserver = QLineEdit(self) + self.imapserver.setToolTip(tooltip) + self.imapserver.setText(prefs['imapserver']) + self.l.addWidget(self.imapserver,row,1) + row+=1 + + label = QLabel(_('IMAP User Name')) + tooltip = _("Name of IMAP user. Eg: yourname@gmail.com\nNote that Gmail addresses need to have IMAP enabled in Gmail Settings first.") + label.setToolTip(tooltip) + self.l.addWidget(label,row,0) + self.imapuser = QLineEdit(self) + self.imapuser.setToolTip(tooltip) + self.imapuser.setText(prefs['imapuser']) + self.l.addWidget(self.imapuser,row,1) + row+=1 + + label = QLabel(_('IMAP User Password')) + tooltip = _("IMAP password. If left empty, FFDL will ask you for your password when you .") + label.setToolTip(tooltip) + self.l.addWidget(label,row,0) + self.imappass = QLineEdit(self) + self.imappass.setToolTip(tooltip) + self.imappass.setEchoMode(QLineEdit.Password) + self.imappass.setText(prefs['imappass']) + self.l.addWidget(self.imappass,row,1) + row+=1 + + self.imapsessionpass = QCheckBox(_('Remember Password for Session (when not entered above)'),self) + self.imapsessionpass.setToolTip(_('If checked, and no password is entered above, FFDL will remember your password until you close calibre or change libraries.')) + self.imapsessionpass.setChecked(prefs['imapsessionpass']) + self.l.addWidget(self.imapsessionpass,row,0,1,-1) + row+=1 + + label = QLabel(_('IMAP Folder Name')) + tooltip = _("Name of IMAP folder to search for new emails. The folder (or label) has to already exist. Use INBOX for your default inbox.") + label.setToolTip(tooltip) + self.l.addWidget(label,row,0) + self.imapfolder = QLineEdit(self) + self.imapfolder.setToolTip(tooltip) + self.imapfolder.setText(prefs['imapfolder']) + self.l.addWidget(self.imapfolder,row,1) + row+=1 + + self.imapmarkread = QCheckBox(_('Mark Emails Read'),self) + self.imapmarkread.setToolTip(_('If checked, emails will be marked as having been read if they contain any story URLs.')) + self.imapmarkread.setChecked(prefs['imapmarkread']) + self.l.addWidget(self.imapmarkread,row,0,1,-1) + row+=1 + + label = QLabel(_("It's safest if you create a separate email account that you use only " + "for your story update notices. FFDL and calibre cannot guarantee that " + "malicious code cannot get your email password once you've entered it. " + "
Use this feature at your own risk. ")) + label.setWordWrap(True) + self.l.addWidget(label,row,0,1,-1,Qt.AlignTop) + self.l.setRowStretch(row,1) + row+=1 + + diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 8e44242..6254cc9 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, print_function) __license__ = 'GPL v3' -__copyright__ = '2014, Jim Miller' +__copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' import traceback, re @@ -1435,3 +1435,49 @@ class ViewLog(SizePersistedDialog): txt = self.tb.toPlainText() QApplication.clipboard().setText(txt) +class EmailPassDialog(QDialog): + ''' + Need to collect Pass for imap. + ''' + def __init__(self, gui, user): + QDialog.__init__(self, gui) + self.status=False + + self.l = QGridLayout() + self.setLayout(self.l) + + self.setWindowTitle(_('Password')) + self.l.addWidget(QLabel(_("Enter Email Password for %s:"%user)),0,0,1,2) + + # self.l.addWidget(QLabel(_("Password:")),1,0) + self.passwd = QLineEdit(self) + self.passwd.setEchoMode(QLineEdit.Password) + self.l.addWidget(self.passwd,1,0,1,2) + + self.ok_button = QPushButton(_('OK'), self) + self.ok_button.clicked.connect(self.ok) + self.l.addWidget(self.ok_button,2,0) + + self.cancel_button = QPushButton(_('Cancel'), self) + self.cancel_button.clicked.connect(self.cancel) + self.l.addWidget(self.cancel_button,2,1) + + # set stretch factors the same. + self.l.setColumnStretch(0,1) + self.l.setColumnStretch(1,1) + + self.resize(self.sizeHint()) + + def ok(self): + self.status=True + self.hide() + + def cancel(self): + self.status=False + self.hide() + + def get_pass(self): + return u"%s"%self.passwd.text() + + def get_remember(self): + return self.remember_pass.isChecked() diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index e40cfe7..9ece4d9 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) __license__ = 'GPL v3' -__copyright__ = '2014, Jim Miller' +__copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' import logging @@ -53,14 +53,14 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html -from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini) from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist) from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( AddNewDialog, UpdateExistingDialog, - LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, + LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog, OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, NotGoingToDownload, RejectUrlEntry ) @@ -135,6 +135,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.menus_lock = threading.RLock() self.menu.aboutToShow.connect(self.about_to_show_menu) + self.imap_pass = None + def initialization_complete(self): # otherwise configured hot keys won't work until the menu's # been displayed once. @@ -231,6 +233,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # We need to reset our menus after switching libraries self.rebuild_menus() rejecturllist.clear_cache() + self.imap_pass = None def rebuild_menus(self): with self.menus_lock: @@ -257,6 +260,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if self.get_epubmerge_plugin(): self.menu.addSeparator() + self.get_list_imap_action = self.create_menu_item_ex(self.menu, _('Get Story URLs to Download from Email'), image='view.png', + unique_name='Get Story URLs from IMAP', + triggered=self.get_urls_from_imap_menu) + self.get_list_url_action = self.create_menu_item_ex(self.menu, _('Get Story URLs to Download from Web Page'), image='view.png', unique_name='Get Story URLs from Web Page', triggered=self.get_urls_from_page_menu) @@ -306,6 +313,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction): triggered=self.list_story_urls) if not self.get_epubmerge_plugin(): + self.get_list_imap_action = self.create_menu_item_ex(self.menu, _('Get Story URLs to Download from Email'), image='view.png', + unique_name='Get Story URLs from IMAP', + triggered=self.get_urls_from_imap_menu) + self.get_list_url_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Web Page'), unique_name='Get Story URLs from Web Page', image='view.png', @@ -385,6 +396,44 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.update_reading_lists(self.gui.library_view.get_selected_ids(),add) + def get_urls_from_imap_menu(self): + + if not prefs['imapserver'] or not prefs['imapuser'] or not prefs['imapfolder']: + info_dialog(self.gui, _('Email Settings not Configured'), + _('FFDL Email Settings are not configured.'), + show=True, + show_copy_button=False) + return + + imap_pass = None + if prefs['imappass']: + imap_pass = prefs['imappass'] + elif self.imap_pass is not None: + imap_pass = self.imap_pass + + if not imap_pass: + d = EmailPassDialog(self.gui,prefs['imapuser']) + d.exec_() + if not d.status: + return + imap_pass = d.get_pass() + if prefs['imapsessionpass']: + self.imap_pass = imap_pass + + url_list = get_urls_from_imap(prefs['imapserver'], + prefs['imapuser'], + imap_pass, + prefs['imapfolder'], + prefs['imapmarkread'],) + + if url_list: + self.add_dialog("\n".join(url_list),merge=False) + else: + info_dialog(self.gui, _('Get Story URLs from Email'), + _('No Valid Story URLs Found in Unread Emails.'), + show=True, + show_copy_button=False) + def get_urls_from_page_menu(self): urltxt = "" diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py index f7df5e6..ebe2175 100644 --- a/calibre-plugin/prefs.py +++ b/calibre-plugin/prefs.py @@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) __license__ = 'GPL v3' -__copyright__ = '2013, Jim Miller' +__copyright__ = '2015, Jim Miller' __docformat__ = 'restructuredtext en' import copy @@ -71,6 +71,13 @@ default_prefs['allow_custcol_from_ini'] = True default_prefs['std_cols_newonly'] = {} +default_prefs['imapserver'] = '' +default_prefs['imapuser'] = '' +default_prefs['imappass'] = '' +default_prefs['imapsessionpass'] = False +default_prefs['imapfolder'] = 'INBOX' +default_prefs['imapmarkread'] = True + # This is where all preferences for this plugin *were* stored # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also # in a global namespace, so make it as unique as possible. diff --git a/fanficdownloader/geturls.py b/fanficdownloader/geturls.py index b66202d..cc614bc 100644 --- a/fanficdownloader/geturls.py +++ b/fanficdownloader/geturls.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2012 Fanficdownloader team +# Copyright 2015 Fanficdownloader team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,9 @@ import re import urlparse import urllib2 as u2 +import imaplib +import email + from BeautifulSoup import BeautifulSoup from gziphttp import GZipProcessor @@ -173,3 +176,55 @@ def form_url(parenturl,url): '','','')) return returl +def get_urls_from_imap(srv,user,passwd,folder,markread=True): + + mail = imaplib.IMAP4_SSL(srv) + mail.login(user, passwd) + mail.list() + # Out: list of "folders" aka labels in gmail. + mail.select(folder) # , readonly=True connect to inbox. + + result, data = mail.uid('search', None, "UNSEEN") + + #print("result:%s"%result) + #print("data:%s"%data) + urls=set() + + #latest_email_uid = data[0].split()[-1] + for email_uid in data[0].split(): + + result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822 + + #print("result:%s"%result) + #print("data:%s"%data) + + raw_email = data[0][1] + + #raw_email = data[0][1] # here's the body, which is raw text of the whole email + # including headers and alternate payloads + + email_message = email.message_from_string(raw_email) + + #print "To:%s"%email_message['To'] + #print "From:%s"%email_message['From'] + #print "Subject:%s"%email_message['Subject'] + + # print("payload:%s"%email_message.get_payload()) + + urllist=[] + for part in email_message.walk(): + #print("part mime:%s"%part.get_content_type()) + if part.get_content_type() == 'text/plain': + urllist.extend(get_urls_from_text(part.get_payload(decode=True))) + if part.get_content_type() == 'text/html': + urllist.extend(get_urls_from_html(part.get_payload(decode=True))) + #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message)) + + if urllist and markread: + #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen') + r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)') + #print("seen result:%s->%s"%(email_uid,r)) + + [ urls.add(x) for x in urllist ] + + return urls