diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py index 9c71284..f51364a 100644 --- a/fanficdownloader/adapters/adapter_adastrafanficcom.py +++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py @@ -90,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php")) diff --git a/fanficdownloader/adapters/adapter_archiveofourownorg.py b/fanficdownloader/adapters/adapter_archiveofourownorg.py index ac54f2e..6aa4ad0 100644 --- a/fanficdownloader/adapters/adapter_archiveofourownorg.py +++ b/fanficdownloader/adapters/adapter_archiveofourownorg.py @@ -163,7 +163,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r"^/works/\d+$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+")) diff --git a/fanficdownloader/adapters/adapter_archiveskyehawkecom.py b/fanficdownloader/adapters/adapter_archiveskyehawkecom.py index 0cbfcbd..6e85338 100644 --- a/fanficdownloader/adapters/adapter_archiveskyehawkecom.py +++ b/fanficdownloader/adapters/adapter_archiveskyehawkecom.py @@ -101,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter): ## Title a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'}) - title=a.text.split('"')[1] + title=stripHTML(a).split('"')[1] self.story.setMetadata('title',title) # Find authorid and URL from... author url. diff --git a/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py b/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py index 9af3bbf..9faf2bf 100644 --- a/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py @@ -157,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter): for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}): a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) if a != None: - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) break diff --git a/fanficdownloader/adapters/adapter_asr3slashzoneorg.py b/fanficdownloader/adapters/adapter_asr3slashzoneorg.py index 56fa9c5..7d0c2a9 100644 --- a/fanficdownloader/adapters/adapter_asr3slashzoneorg.py +++ b/fanficdownloader/adapters/adapter_asr3slashzoneorg.py @@ -128,7 +128,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_bloodtiesfancom.py b/fanficdownloader/adapters/adapter_bloodtiesfancom.py index c9f0eb1..2e5748e 100644 --- a/fanficdownloader/adapters/adapter_bloodtiesfancom.py +++ b/fanficdownloader/adapters/adapter_bloodtiesfancom.py @@ -207,7 +207,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_castlefansorg.py b/fanficdownloader/adapters/adapter_castlefansorg.py index 60f8607..f1fc29d 100644 --- a/fanficdownloader/adapters/adapter_castlefansorg.py +++ b/fanficdownloader/adapters/adapter_castlefansorg.py @@ -180,7 +180,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX pagetitle = soup.find('div',{'id':'pagetitle'}) ## Title a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py index 8160b1e..3c1853e 100644 --- a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py @@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_checkmatedcom.py b/fanficdownloader/adapters/adapter_checkmatedcom.py index 72be1f4..b1ca125 100644 --- a/fanficdownloader/adapters/adapter_checkmatedcom.py +++ b/fanficdownloader/adapters/adapter_checkmatedcom.py @@ -143,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter): # Now go hunting for all the meta data and the chapter list. ## Title - a = soup.findAll('span', {'class' : 'storytitle'}) - self.story.setMetadata('title',a[0].string) + a = soup.find('span', {'class' : 'storytitle'}) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. - a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+")) + a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+")) self.story.setMetadata('authorId',a['href'].split('=')[1]) self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href']) self.story.setMetadata('author',a.string) diff --git a/fanficdownloader/adapters/adapter_destinysgatewaycom.py b/fanficdownloader/adapters/adapter_destinysgatewaycom.py index 61b300a..adf41d2 100644 --- a/fanficdownloader/adapters/adapter_destinysgatewaycom.py +++ b/fanficdownloader/adapters/adapter_destinysgatewaycom.py @@ -131,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_dokugacom.py b/fanficdownloader/adapters/adapter_dokugacom.py index 826b7c9..6a21e80 100644 --- a/fanficdownloader/adapters/adapter_dokugacom.py +++ b/fanficdownloader/adapters/adapter_dokugacom.py @@ -164,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter): aut.extract() a = a.string[:(len(a.string)-4)] - self.story.setMetadata('title',a) + self.story.setMetadata('title',stripHTML(a)) # Find the chapters: chapters = soup.find('select').findAll('option') diff --git a/fanficdownloader/adapters/adapter_dotmoonnet.py b/fanficdownloader/adapters/adapter_dotmoonnet.py index af6cfa3..e2dedb7 100644 --- a/fanficdownloader/adapters/adapter_dotmoonnet.py +++ b/fanficdownloader/adapters/adapter_dotmoonnet.py @@ -141,7 +141,7 @@ class DotMoonNetAdapter(BaseSiteAdapter): ## Title a = body.find('b') - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1' a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+")) diff --git a/fanficdownloader/adapters/adapter_dracoandginnycom.py b/fanficdownloader/adapters/adapter_dracoandginnycom.py index c1b5b5d..f666249 100644 --- a/fanficdownloader/adapters/adapter_dracoandginnycom.py +++ b/fanficdownloader/adapters/adapter_dracoandginnycom.py @@ -170,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_dramioneorg.py b/fanficdownloader/adapters/adapter_dramioneorg.py index f3b4a46..6dd0a39 100644 --- a/fanficdownloader/adapters/adapter_dramioneorg.py +++ b/fanficdownloader/adapters/adapter_dramioneorg.py @@ -154,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_efictionestelielde.py b/fanficdownloader/adapters/adapter_efictionestelielde.py index 8036278..4343a2b 100644 --- a/fanficdownloader/adapters/adapter_efictionestelielde.py +++ b/fanficdownloader/adapters/adapter_efictionestelielde.py @@ -104,7 +104,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter): pagetitle = soup.find('div',{'id':'pagetitle'}) ## Title a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py index 4602ae7..6835212 100644 --- a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py @@ -132,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_fictionalleyorg.py b/fanficdownloader/adapters/adapter_fictionalleyorg.py index 67787aa..b9fb449 100644 --- a/fanficdownloader/adapters/adapter_fictionalleyorg.py +++ b/fanficdownloader/adapters/adapter_fictionalleyorg.py @@ -111,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter): url=self.url logger.debug("Normalizing to URL: "+url) ## title's right there... - self.story.setMetadata('title',storya.string) + self.story.setMetadata('title',stripHTML(storya)) data = self._fetchUrl(url) soup = bs.BeautifulSoup(data) chapterlinklist = soup.findAll('a',{'class':'chapterlink'}) @@ -121,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter): ## same way. chapsoup = bs.BeautifulSoup(chapterdata) storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1] - self.story.setMetadata('title',storya.string) + self.story.setMetadata('title',stripHTML(storya)) del chapsoup del chapterdata diff --git a/fanficdownloader/adapters/adapter_ficwadcom.py b/fanficdownloader/adapters/adapter_ficwadcom.py index 7c16fd7..4446df7 100644 --- a/fanficdownloader/adapters/adapter_ficwadcom.py +++ b/fanficdownloader/adapters/adapter_ficwadcom.py @@ -116,7 +116,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter): # title - first h4 tag will be title. titleh4 = soup.find('h4') - self.story.setMetadata('title', titleh4.a.string) + self.story.setMetadata('title', stripHTML(titleh4.a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"^/author/\d+")) diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index d337950..b5cd487 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -117,7 +117,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId'))) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # self.story.setMetadata("title", storyMetadata["title"]) # if not storyMetadata["title"]: diff --git a/fanficdownloader/adapters/adapter_finestoriescom.py b/fanficdownloader/adapters/adapter_finestoriescom.py index 2bb4239..c5b626b 100644 --- a/fanficdownloader/adapters/adapter_finestoriescom.py +++ b/fanficdownloader/adapters/adapter_finestoriescom.py @@ -142,7 +142,7 @@ class FineStoriesComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"/a/\w+")) diff --git a/fanficdownloader/adapters/adapter_grangerenchantedcom.py b/fanficdownloader/adapters/adapter_grangerenchantedcom.py index cb9caf7..a39f738 100644 --- a/fanficdownloader/adapters/adapter_grangerenchantedcom.py +++ b/fanficdownloader/adapters/adapter_grangerenchantedcom.py @@ -182,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py index 7a012dc..2fc64dc 100644 --- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py @@ -92,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId'))) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995' if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")): raise exceptions.AdultCheckRequired(self.url) diff --git a/fanficdownloader/adapters/adapter_hennethannunnet.py b/fanficdownloader/adapters/adapter_hennethannunnet.py index 015fdcf..340e8c2 100644 --- a/fanficdownloader/adapters/adapter_hennethannunnet.py +++ b/fanficdownloader/adapters/adapter_hennethannunnet.py @@ -101,7 +101,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter): ## Title a = soup.find('h2', {'id':'page_heading'}) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")): diff --git a/fanficdownloader/adapters/adapter_hlfictionnet.py b/fanficdownloader/adapters/adapter_hlfictionnet.py index c7edf4b..5d15f7d 100644 --- a/fanficdownloader/adapters/adapter_hlfictionnet.py +++ b/fanficdownloader/adapters/adapter_hlfictionnet.py @@ -106,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter): self.story.setMetadata('author',aut.string) aut.extract() - self.story.setMetadata('title',a.string[:(len(a.string)-3)]) + self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)]) # Find the chapters: chapters=soup.find('select') diff --git a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py index a7b8fee..8a57e35 100644 --- a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py +++ b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py @@ -105,7 +105,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_iketernalnet.py b/fanficdownloader/adapters/adapter_iketernalnet.py index dd2aa1b..a89ff8f 100644 --- a/fanficdownloader/adapters/adapter_iketernalnet.py +++ b/fanficdownloader/adapters/adapter_iketernalnet.py @@ -183,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_imagineeficcom.py b/fanficdownloader/adapters/adapter_imagineeficcom.py index b524fdd..e080bef 100644 --- a/fanficdownloader/adapters/adapter_imagineeficcom.py +++ b/fanficdownloader/adapters/adapter_imagineeficcom.py @@ -170,7 +170,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_indeathnet.py b/fanficdownloader/adapters/adapter_indeathnet.py index 4988bef..4d6fd93 100644 --- a/fanficdownloader/adapters/adapter_indeathnet.py +++ b/fanficdownloader/adapters/adapter_indeathnet.py @@ -109,7 +109,7 @@ class InDeathNetAdapter(BaseSiteAdapter): ## Title h = soup.find('a', id="blog_title") t = h.find('span') - self.story.setMetadata('title',t.contents[0].string.strip()) + self.story.setMetadata('title',stripHTML(t.contents[0]).strip()) s = t.find('div') if s != None: @@ -122,9 +122,6 @@ class InDeathNetAdapter(BaseSiteAdapter): self.story.setMetadata('authorUrl',a['href']) self.story.setMetadata('author',m.group('name')) - - - # Find the chapters: chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$")) diff --git a/fanficdownloader/adapters/adapter_libraryofmoriacom.py b/fanficdownloader/adapters/adapter_libraryofmoriacom.py index 57083b2..6f8ff7f 100644 --- a/fanficdownloader/adapters/adapter_libraryofmoriacom.py +++ b/fanficdownloader/adapters/adapter_libraryofmoriacom.py @@ -129,7 +129,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py index 07c416b..93b8b76 100644 --- a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py @@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_mediaminerorg.py b/fanficdownloader/adapters/adapter_mediaminerorg.py index 93d7abe..b9c4641 100644 --- a/fanficdownloader/adapters/adapter_mediaminerorg.py +++ b/fanficdownloader/adapters/adapter_mediaminerorg.py @@ -107,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter): for font in title.findAll('font'): font.extract() # removes 'font' tags from inside the td. if title.has_key('colspan'): - titlet = title.text + titlet = stripHTML(title) else: ## No colspan, it's part chapter title--even if it's a one-shot. - titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title + titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title self.story.setMetadata('title',titlet) ## The story title is difficult to reliably parse from the ## story pages. Getting it from the author page is, but costs diff --git a/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py index 480e454..91b8d6e 100644 --- a/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py +++ b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py @@ -170,7 +170,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_mugglenetcom.py b/fanficdownloader/adapters/adapter_mugglenetcom.py index d3bf5a2..31599fb 100644 --- a/fanficdownloader/adapters/adapter_mugglenetcom.py +++ b/fanficdownloader/adapters/adapter_mugglenetcom.py @@ -193,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_nationallibrarynet.py b/fanficdownloader/adapters/adapter_nationallibrarynet.py index c4ad08d..7e1947d 100644 --- a/fanficdownloader/adapters/adapter_nationallibrarynet.py +++ b/fanficdownloader/adapters/adapter_nationallibrarynet.py @@ -104,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter): ## Title a = soup.find('h1') - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+")) diff --git a/fanficdownloader/adapters/adapter_ncisficcom.py b/fanficdownloader/adapters/adapter_ncisficcom.py index 85d7a6e..4e6a448 100644 --- a/fanficdownloader/adapters/adapter_ncisficcom.py +++ b/fanficdownloader/adapters/adapter_ncisficcom.py @@ -103,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter): ## Title a = soup.find('h1') - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+")) diff --git a/fanficdownloader/adapters/adapter_netraptororg.py b/fanficdownloader/adapters/adapter_netraptororg.py index 86082f4..524d807 100644 --- a/fanficdownloader/adapters/adapter_netraptororg.py +++ b/fanficdownloader/adapters/adapter_netraptororg.py @@ -97,7 +97,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter): ## Title pagetitle = soup.find('div',{'id':'pagetitle'}) a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_nfacommunitycom.py b/fanficdownloader/adapters/adapter_nfacommunitycom.py index 01cdb42..98aa215 100644 --- a/fanficdownloader/adapters/adapter_nfacommunitycom.py +++ b/fanficdownloader/adapters/adapter_nfacommunitycom.py @@ -156,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_nhamagicalworldsus.py b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py index f8574f9..91fc965 100644 --- a/fanficdownloader/adapters/adapter_nhamagicalworldsus.py +++ b/fanficdownloader/adapters/adapter_nhamagicalworldsus.py @@ -115,7 +115,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter): a = info.find('a') if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \ ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']: - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) break diff --git a/fanficdownloader/adapters/adapter_nickandgregnet.py b/fanficdownloader/adapters/adapter_nickandgregnet.py index 9ece5a8..9a19c11 100644 --- a/fanficdownloader/adapters/adapter_nickandgregnet.py +++ b/fanficdownloader/adapters/adapter_nickandgregnet.py @@ -101,7 +101,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py index 2dc80b2..475c87c 100644 --- a/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py @@ -158,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter): for info in asoup.findAll('table', {'class' : 'border'}): a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) if a != None: - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) break diff --git a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py index 596244b..dc67968 100644 --- a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py @@ -158,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_phoenixsongnet.py b/fanficdownloader/adapters/adapter_phoenixsongnet.py index 9de6bb8..f1da893 100644 --- a/fanficdownloader/adapters/adapter_phoenixsongnet.py +++ b/fanficdownloader/adapters/adapter_phoenixsongnet.py @@ -136,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter): ## Title b = soup.find('div', {'id' : 'nav25'}) a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. /fanfiction/stories.php?psid=125 a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+")) diff --git a/fanficdownloader/adapters/adapter_pommedesangcom.py b/fanficdownloader/adapters/adapter_pommedesangcom.py index 15edf9e..ffa7f8f 100644 --- a/fanficdownloader/adapters/adapter_pommedesangcom.py +++ b/fanficdownloader/adapters/adapter_pommedesangcom.py @@ -180,7 +180,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+')) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py index 9a90aa4..b08b542 100644 --- a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py +++ b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py @@ -136,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py index 4427e9b..58624e6 100644 --- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -83,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_potterficscom.py b/fanficdownloader/adapters/adapter_potterficscom.py index 730a898..84b9d22 100644 --- a/fanficdownloader/adapters/adapter_potterficscom.py +++ b/fanficdownloader/adapters/adapter_potterficscom.py @@ -147,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter): #find first metadata block mb = cell.div.findNextSibling('div') #Get meta... - self.story.setMetadata('title', mb.b.string) + self.story.setMetadata('title', stripHTML(mb.b)) #strip out brackets on rating self.story.setMetadata('rating', mb.span.string[1:-1]) #Completion status is denoted by the presence of this image: diff --git a/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py b/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py index d387474..7fc36d8 100644 --- a/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py +++ b/fanficdownloader/adapters/adapter_potterheadsanonymouscom.py @@ -181,7 +181,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter): ## Title a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_pretendercentrecom.py b/fanficdownloader/adapters/adapter_pretendercentrecom.py index ef65b71..4097ecc 100644 --- a/fanficdownloader/adapters/adapter_pretendercentrecom.py +++ b/fanficdownloader/adapters/adapter_pretendercentrecom.py @@ -134,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_psychficcom.py b/fanficdownloader/adapters/adapter_psychficcom.py index 33f53d8..bb69eea 100644 --- a/fanficdownloader/adapters/adapter_psychficcom.py +++ b/fanficdownloader/adapters/adapter_psychficcom.py @@ -130,7 +130,7 @@ class PsychFicComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_qafficcom.py b/fanficdownloader/adapters/adapter_qafficcom.py index f557351..ce95780 100644 --- a/fanficdownloader/adapters/adapter_qafficcom.py +++ b/fanficdownloader/adapters/adapter_qafficcom.py @@ -138,7 +138,7 @@ class QafFicComAdapter(BaseSiteAdapter): self.story.setMetadata('author',aut.string) aut.extract() - self.story.setMetadata('title',a.string[:(len(a.string)-3)]) + self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)]) # Find the chapters: chapters=soup.find('select') diff --git a/fanficdownloader/adapters/adapter_restrictedsectionorg.py b/fanficdownloader/adapters/adapter_restrictedsectionorg.py index ce02ca0..686e053 100644 --- a/fanficdownloader/adapters/adapter_restrictedsectionorg.py +++ b/fanficdownloader/adapters/adapter_restrictedsectionorg.py @@ -126,10 +126,12 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter): self.story.setMetadata('authorId',ahref.split('=')[1]) self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref) - self.story.setMetadata('author',a.string) + self.story.setMetadata('author',stripHTML(a)) # title, remove byauthorname. - self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)]) + auth=stripHTML(a) + title=stripHTML(h2) + self.story.setMetadata('title',title[:title.index(" by "+auth)]) dates = soup.findAll('span', {'class':'date'}) if dates: # only for multi-chapter diff --git a/fanficdownloader/adapters/adapter_samdeanarchivenu.py b/fanficdownloader/adapters/adapter_samdeanarchivenu.py index c0549a5..3c2bf27 100644 --- a/fanficdownloader/adapters/adapter_samdeanarchivenu.py +++ b/fanficdownloader/adapters/adapter_samdeanarchivenu.py @@ -110,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter): self.story.setMetadata('author',aut.string) aut.extract() - self.story.setMetadata('title',a.string[:(len(a.string)-3)]) + self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)]) # Find the chapters: chapters=soup.find('select') diff --git a/fanficdownloader/adapters/adapter_scarheadnet.py b/fanficdownloader/adapters/adapter_scarheadnet.py index 736c047..593abc6 100644 --- a/fanficdownloader/adapters/adapter_scarheadnet.py +++ b/fanficdownloader/adapters/adapter_scarheadnet.py @@ -181,7 +181,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter): ## Title a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py index 7267599..3a46bb7 100644 --- a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py +++ b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py @@ -139,7 +139,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py index b8a9e22..aba87c0 100644 --- a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py +++ b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py @@ -140,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_simplyundeniablecom.py b/fanficdownloader/adapters/adapter_simplyundeniablecom.py index 990e372..2277ac3 100644 --- a/fanficdownloader/adapters/adapter_simplyundeniablecom.py +++ b/fanficdownloader/adapters/adapter_simplyundeniablecom.py @@ -143,7 +143,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter): ## Title a = soup.find('h1') - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) @@ -155,7 +155,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter): for info in asoup.findAll('table', {'cellpadding' : '5'}): a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) if a != None: - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) break # Find the chapters: diff --git a/fanficdownloader/adapters/adapter_sinfuldesireorg.py b/fanficdownloader/adapters/adapter_sinfuldesireorg.py index d6a96a7..c1e9820 100644 --- a/fanficdownloader/adapters/adapter_sinfuldesireorg.py +++ b/fanficdownloader/adapters/adapter_sinfuldesireorg.py @@ -130,7 +130,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_squidgeorgpeja.py b/fanficdownloader/adapters/adapter_squidgeorgpeja.py index ba2d3e6..b79bb6c 100644 --- a/fanficdownloader/adapters/adapter_squidgeorgpeja.py +++ b/fanficdownloader/adapters/adapter_squidgeorgpeja.py @@ -122,7 +122,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter): # formating, so it's a little ugly. title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')}) - self.story.setMetadata('title',title.string) + self.story.setMetadata('title',stripHTML(title)) titleblock=title.parent.parent chapterselect=soup.find('select',{'name':'chapter'}) diff --git a/fanficdownloader/adapters/adapter_stargateatlantisorg.py b/fanficdownloader/adapters/adapter_stargateatlantisorg.py index 612f995..9fd3d0a 100644 --- a/fanficdownloader/adapters/adapter_stargateatlantisorg.py +++ b/fanficdownloader/adapters/adapter_stargateatlantisorg.py @@ -106,7 +106,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter): self.story.setMetadata('author',aut.string) aut.extract() - self.story.setMetadata('title',a.string[:(len(a.string)-3)]) + self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)]) # Find the chapters: chapters=soup.findAll('div', {'class' : 'content'}) diff --git a/fanficdownloader/adapters/adapter_storiesofardacom.py b/fanficdownloader/adapters/adapter_storiesofardacom.py index acdddda..0dfc4a8 100644 --- a/fanficdownloader/adapters/adapter_storiesofardacom.py +++ b/fanficdownloader/adapters/adapter_storiesofardacom.py @@ -104,7 +104,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter): asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) a.find('em').extract() - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find the chapters: chapterview.asp?sid=7000&cid=30919 chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$")) diff --git a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py index f724d39..90c3714 100644 --- a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py +++ b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py @@ -133,7 +133,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId'))) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_thealphagatecom.py b/fanficdownloader/adapters/adapter_thealphagatecom.py index fa28e40..ce5abec 100644 --- a/fanficdownloader/adapters/adapter_thealphagatecom.py +++ b/fanficdownloader/adapters/adapter_thealphagatecom.py @@ -99,7 +99,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_thehookupzonenet.py b/fanficdownloader/adapters/adapter_thehookupzonenet.py index 618c21e..22636b8 100644 --- a/fanficdownloader/adapters/adapter_thehookupzonenet.py +++ b/fanficdownloader/adapters/adapter_thehookupzonenet.py @@ -179,7 +179,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_themasquenet.py b/fanficdownloader/adapters/adapter_themasquenet.py index 2be75ff..e2e80ff 100644 --- a/fanficdownloader/adapters/adapter_themasquenet.py +++ b/fanficdownloader/adapters/adapter_themasquenet.py @@ -177,7 +177,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py b/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py index 159a946..e0a8c4f 100644 --- a/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py +++ b/fanficdownloader/adapters/adapter_thepetulantpoetesscom.py @@ -165,7 +165,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter): for info in asoup.findAll('td', {'class' : 'highlightcolor1'}): a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) if a != None: - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) found = 1 break index=index+1 diff --git a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py index bdf26d2..98bc028 100644 --- a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py +++ b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py @@ -165,7 +165,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId'))) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_tokrafandomnetcom.py b/fanficdownloader/adapters/adapter_tokrafandomnetcom.py index 3576f97..70a1d90 100644 --- a/fanficdownloader/adapters/adapter_tokrafandomnetcom.py +++ b/fanficdownloader/adapters/adapter_tokrafandomnetcom.py @@ -131,7 +131,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_twilightarchivescom.py b/fanficdownloader/adapters/adapter_twilightarchivescom.py index 99a5c7e..7995456 100644 --- a/fanficdownloader/adapters/adapter_twilightarchivescom.py +++ b/fanficdownloader/adapters/adapter_twilightarchivescom.py @@ -100,7 +100,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter): ## Title a = soup.find('h1') - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find the chapters: chapters=soup.find('ol', {'class' : 'chapters'}) diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index c45b3e9..f67e985 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -130,7 +130,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php")) diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py index 38ce580..d88d932 100644 --- a/fanficdownloader/adapters/adapter_twiwritenet.py +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -144,7 +144,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter): ## Title a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_walkingtheplankorg.py b/fanficdownloader/adapters/adapter_walkingtheplankorg.py index 890fc64..11f4fc4 100644 --- a/fanficdownloader/adapters/adapter_walkingtheplankorg.py +++ b/fanficdownloader/adapters/adapter_walkingtheplankorg.py @@ -115,7 +115,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter): ## Title a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_whoficcom.py b/fanficdownloader/adapters/adapter_whoficcom.py index 4030bc5..e43f8d5 100644 --- a/fanficdownloader/adapters/adapter_whoficcom.py +++ b/fanficdownloader/adapters/adapter_whoficcom.py @@ -70,7 +70,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter): raise e # pull title(title) and author from the HTML title. - title = soup.find('title').string + title = stripHTML(soup.find('title')) logger.debug('Title: %s' % title) title = title.split('::')[1].strip() self.story.setMetadata('title',title.split(' by ')[0].strip()) diff --git a/fanficdownloader/adapters/adapter_wizardtalesnet.py b/fanficdownloader/adapters/adapter_wizardtalesnet.py index 3054ea6..3f60560 100644 --- a/fanficdownloader/adapters/adapter_wizardtalesnet.py +++ b/fanficdownloader/adapters/adapter_wizardtalesnet.py @@ -171,7 +171,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_wolverineandroguecom.py b/fanficdownloader/adapters/adapter_wolverineandroguecom.py index cf16068..4bdfb91 100644 --- a/fanficdownloader/adapters/adapter_wolverineandroguecom.py +++ b/fanficdownloader/adapters/adapter_wolverineandroguecom.py @@ -101,7 +101,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.text) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+")) diff --git a/fanficdownloader/adapters/adapter_wraithbaitcom.py b/fanficdownloader/adapters/adapter_wraithbaitcom.py index 0b17a2e..ab31c9f 100644 --- a/fanficdownloader/adapters/adapter_wraithbaitcom.py +++ b/fanficdownloader/adapters/adapter_wraithbaitcom.py @@ -113,7 +113,7 @@ class WraithBaitComAdapter(BaseSiteAdapter): ## Title pt = soup.find('div', {'id' : 'pagetitle'}) a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$")) - self.story.setMetadata('title',a.string) + self.story.setMetadata('title',stripHTML(a)) # Find authorid and URL from... author url. alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))