From 48c81b1d1e940140660d3b361a3fa8df20a6b141 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 19 Mar 2013 14:21:14 -0500 Subject: [PATCH] Add seriesUrl (and generated seriesHTML) as valid metadata entries. --- defaults.ini | 15 +++++++++------ .../adapters/adapter_adastrafanficcom.py | 1 + .../adapters/adapter_archiveofourownorg.py | 1 + .../adapters/adapter_bloodtiesfancom.py | 1 + .../adapters/adapter_castlefansorg.py | 1 + .../adapters/adapter_chaossycophanthexcom.py | 1 + .../adapters/adapter_darksolaceorg.py | 1 + .../adapters/adapter_destinysgatewaycom.py | 1 + .../adapters/adapter_dracoandginnycom.py | 1 + fanficdownloader/adapters/adapter_dramioneorg.py | 1 + .../adapters/adapter_efictionestelielde.py | 1 + fanficdownloader/adapters/adapter_efpfanficnet.py | 1 + .../adapter_erosnsapphosycophanthexcom.py | 1 + .../adapters/adapter_finestoriescom.py | 1 + .../adapters/adapter_grangerenchantedcom.py | 1 + fanficdownloader/adapters/adapter_hlfictionnet.py | 1 + .../adapters/adapter_hpfanficarchivecom.py | 1 + .../adapters/adapter_imagineeficcom.py | 1 + .../adapters/adapter_jlaunlimitedcom.py | 1 + fanficdownloader/adapters/adapter_ksarchivecom.py | 1 + .../adapters/adapter_libraryofmoriacom.py | 1 + .../adapters/adapter_lumossycophanthexcom.py | 1 + .../adapters/adapter_merlinficdtwinscouk.py | 1 + .../adapters/adapter_midnightwhispersca.py | 1 + fanficdownloader/adapters/adapter_mugglenetcom.py | 1 + .../adapters/adapter_nationallibrarynet.py | 1 + fanficdownloader/adapters/adapter_ncisficcom.py | 1 + .../adapters/adapter_ncisfictionnet.py | 2 +- .../adapters/adapter_nfacommunitycom.py | 1 + .../adapters/adapter_onedirectionfanfictioncom.py | 1 + .../adapters/adapter_pommedesangcom.py | 1 + .../adapters/adapter_ponyfictionarchivenet.py | 1 + .../adapters/adapter_potionsandsnitchesnet.py | 1 + .../adapters/adapter_pretendercentrecom.py | 1 + .../adapters/adapter_prisonbreakficnet.py | 1 + fanficdownloader/adapters/adapter_psychficcom.py | 1 + fanficdownloader/adapters/adapter_qafficcom.py | 1 + .../adapters/adapter_samdeanarchivenu.py | 1 + .../adapters/adapter_scarvesandcoffeenet.py | 1 + .../adapters/adapter_sg1heliopoliscom.py | 1 + .../adapters/adapter_sinfuldesireorg.py | 1 + fanficdownloader/adapters/adapter_siyecouk.py | 1 + .../adapters/adapter_squidgeorgpeja.py | 1 + .../adapters/adapter_stargateatlantisorg.py | 1 + .../adapters/adapter_svufictioncom.py | 1 + .../adapters/adapter_tenhawkpresentscom.py | 1 + fanficdownloader/adapters/adapter_test1.py | 2 ++ .../adapters/adapter_thealphagatecom.py | 1 + .../adapters/adapter_thehookupzonenet.py | 1 + .../adapters/adapter_thequidditchpitchorg.py | 1 + .../adapters/adapter_thewriterscoffeeshopcom.py | 1 + fanficdownloader/adapters/adapter_tthfanficorg.py | 4 +--- .../adapters/adapter_twilightarchivescom.py | 1 + .../adapters/adapter_twilightednet.py | 1 + fanficdownloader/adapters/adapter_twiwritenet.py | 1 + .../adapters/adapter_walkingtheplankorg.py | 1 + fanficdownloader/adapters/adapter_whoficcom.py | 1 + .../adapters/adapter_wizardtalesnet.py | 1 + .../adapters/adapter_wolverineandroguecom.py | 1 + .../adapters/adapter_wraithbaitcom.py | 1 + fanficdownloader/configurable.py | 2 ++ fanficdownloader/story.py | 10 ++++++---- 62 files changed, 77 insertions(+), 14 deletions(-) diff --git a/defaults.ini b/defaults.ini index bd18b80..6ec13bc 100644 --- a/defaults.ini +++ b/defaults.ini @@ -44,6 +44,9 @@ language_label:Language characters_label:Characters ships_label:Relationships series_label:Series +seriesUrl_label:Series URL +## seriesHTML is series as a link to seriesUrl. +seriesHTML_label:Series ## Completed/In-Progress status_label:Status ## Dates story first published, last updated, and downloaded(last with time). @@ -81,7 +84,7 @@ dateUpdated_format:%%Y-%%m-%%d ## You can include extra text or HTML that will be included as-is in ## the title page. Eg: titlepage_entries: ...,
,summary,
,... ## All current formats already include title and author. -titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description +titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description ## Try to collect series name and number of this story in series. ## Some sites (ab)use 'series' for reading lists and personal @@ -270,7 +273,7 @@ output_css: [txt] ## Add URLs since there aren't links. -titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description +titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description ## Width to word wrap text output. 0 indicates no wrapping. wrap_width: 78 @@ -304,13 +307,13 @@ include_logpage: false ## if in the list. You can include extra text or HTML that will be ## included as-is in each log entry. Eg: logpage_entries: ...,
, ## summary,
,... -logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings +logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings ## epub->mobi conversions typically don't like tables. titlepage_use_table: false ## When using tables, make these span both columns. -wide_titlepage_entries: description, storyUrl, author URL +wide_titlepage_entries: description, storyUrl, authorUrl, seriesUrl ## output background color--only used by html and epub (and ignored in ## epub by many readers). Included below in output_css--will be @@ -1219,12 +1222,12 @@ extraships:Harry Potter/Ginny Weasley # www.squidge.org/peja calls it Fandom category_label:Fandom # Remove numWords -- www.squidge.org/peja word counts are inaccurate -titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description +titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description [www.squidge.org/peja:txt] ## Add URLs since there aren't links. # Remove numWords -- www.squidge.org/peja word counts are inaccurate -titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description +titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description [www.storiesofarda.com] ## Site dedicated to these categories/characters/ships diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py index aacd783..e58c972 100644 --- a/fanficdownloader/adapters/adapter_adastrafanficcom.py +++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py @@ -195,6 +195,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_archiveofourownorg.py b/fanficdownloader/adapters/adapter_archiveofourownorg.py index 089e555..0de4771 100644 --- a/fanficdownloader/adapters/adapter_archiveofourownorg.py +++ b/fanficdownloader/adapters/adapter_archiveofourownorg.py @@ -295,6 +295,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter): series_url = 'http://'+self.host+'/fanfic/'+b['href'] series_index = int(a.text.split(' ')[1]) self.setSeries(series_name, series_index) + self.story.setMetadata('seriesUrl',series_url) except: # I find it hard to care if the series parsing fails diff --git a/fanficdownloader/adapters/adapter_bloodtiesfancom.py b/fanficdownloader/adapters/adapter_bloodtiesfancom.py index 98c32cc..1c687ba 100644 --- a/fanficdownloader/adapters/adapter_bloodtiesfancom.py +++ b/fanficdownloader/adapters/adapter_bloodtiesfancom.py @@ -311,6 +311,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_castlefansorg.py b/fanficdownloader/adapters/adapter_castlefansorg.py index 9345d24..a7ec99a 100644 --- a/fanficdownloader/adapters/adapter_castlefansorg.py +++ b/fanficdownloader/adapters/adapter_castlefansorg.py @@ -284,6 +284,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py index 2436630..d78b89b 100644 --- a/fanficdownloader/adapters/adapter_chaossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_chaossycophanthexcom.py @@ -212,6 +212,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_darksolaceorg.py b/fanficdownloader/adapters/adapter_darksolaceorg.py index f58f180..6b671d9 100644 --- a/fanficdownloader/adapters/adapter_darksolaceorg.py +++ b/fanficdownloader/adapters/adapter_darksolaceorg.py @@ -252,6 +252,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_destinysgatewaycom.py b/fanficdownloader/adapters/adapter_destinysgatewaycom.py index f581f50..acdc5d1 100644 --- a/fanficdownloader/adapters/adapter_destinysgatewaycom.py +++ b/fanficdownloader/adapters/adapter_destinysgatewaycom.py @@ -218,6 +218,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_dracoandginnycom.py b/fanficdownloader/adapters/adapter_dracoandginnycom.py index ffc7457..b0dedda 100644 --- a/fanficdownloader/adapters/adapter_dracoandginnycom.py +++ b/fanficdownloader/adapters/adapter_dracoandginnycom.py @@ -273,6 +273,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_dramioneorg.py b/fanficdownloader/adapters/adapter_dramioneorg.py index 0b8f6f0..9cd7574 100644 --- a/fanficdownloader/adapters/adapter_dramioneorg.py +++ b/fanficdownloader/adapters/adapter_dramioneorg.py @@ -274,6 +274,7 @@ class DramioneOrgAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_efictionestelielde.py b/fanficdownloader/adapters/adapter_efictionestelielde.py index 0eccd7d..b45eb9f 100644 --- a/fanficdownloader/adapters/adapter_efictionestelielde.py +++ b/fanficdownloader/adapters/adapter_efictionestelielde.py @@ -195,6 +195,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter): name=seriessoup.find('div', {'id' : 'pagetitle'}) name.find('a').extract() self.setSeries(name.text.split(' by[')[0], i) + self.story.setMetadata('seriesUrl',series_url) i=0 break i+=1 diff --git a/fanficdownloader/adapters/adapter_efpfanficnet.py b/fanficdownloader/adapters/adapter_efpfanficnet.py index 2c38f1f..0cb3033 100644 --- a/fanficdownloader/adapters/adapter_efpfanficnet.py +++ b/fanficdownloader/adapters/adapter_efpfanficnet.py @@ -280,6 +280,7 @@ class EFPFanFicNet(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1': self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py index 80cee30..ff37cec 100644 --- a/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py @@ -230,6 +230,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_finestoriescom.py b/fanficdownloader/adapters/adapter_finestoriescom.py index efdda34..ea0e825 100644 --- a/fanficdownloader/adapters/adapter_finestoriescom.py +++ b/fanficdownloader/adapters/adapter_finestoriescom.py @@ -187,6 +187,7 @@ class FineStoriesComAdapter(BaseSiteAdapter): a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+")) i = a.parent.text.split('(')[1].split(')')[0] self.setSeries(a.text, i) + self.story.setMetadata('seriesUrl','http://'+self.host+a['href']) except: pass try: diff --git a/fanficdownloader/adapters/adapter_grangerenchantedcom.py b/fanficdownloader/adapters/adapter_grangerenchantedcom.py index dd4723f..b0876ec 100644 --- a/fanficdownloader/adapters/adapter_grangerenchantedcom.py +++ b/fanficdownloader/adapters/adapter_grangerenchantedcom.py @@ -275,6 +275,7 @@ class GrangerEnchantedCom(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_hlfictionnet.py b/fanficdownloader/adapters/adapter_hlfictionnet.py index 48c0cd0..14bc335 100644 --- a/fanficdownloader/adapters/adapter_hlfictionnet.py +++ b/fanficdownloader/adapters/adapter_hlfictionnet.py @@ -206,6 +206,7 @@ class HLFictionNetAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py index 1c263bd..2bf5fff 100644 --- a/fanficdownloader/adapters/adapter_hpfanficarchivecom.py +++ b/fanficdownloader/adapters/adapter_hpfanficarchivecom.py @@ -193,6 +193,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_imagineeficcom.py b/fanficdownloader/adapters/adapter_imagineeficcom.py index 2b69942..b331055 100644 --- a/fanficdownloader/adapters/adapter_imagineeficcom.py +++ b/fanficdownloader/adapters/adapter_imagineeficcom.py @@ -264,6 +264,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_jlaunlimitedcom.py b/fanficdownloader/adapters/adapter_jlaunlimitedcom.py index ded95dc..f9e57ee 100644 --- a/fanficdownloader/adapters/adapter_jlaunlimitedcom.py +++ b/fanficdownloader/adapters/adapter_jlaunlimitedcom.py @@ -234,6 +234,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_ksarchivecom.py b/fanficdownloader/adapters/adapter_ksarchivecom.py index ff979fb..f76b8e6 100644 --- a/fanficdownloader/adapters/adapter_ksarchivecom.py +++ b/fanficdownloader/adapters/adapter_ksarchivecom.py @@ -285,6 +285,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_libraryofmoriacom.py b/fanficdownloader/adapters/adapter_libraryofmoriacom.py index 68c3c3a..afad729 100644 --- a/fanficdownloader/adapters/adapter_libraryofmoriacom.py +++ b/fanficdownloader/adapters/adapter_libraryofmoriacom.py @@ -225,6 +225,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py index dd0b0d0..a64bac0 100644 --- a/fanficdownloader/adapters/adapter_lumossycophanthexcom.py +++ b/fanficdownloader/adapters/adapter_lumossycophanthexcom.py @@ -212,6 +212,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py index 98fd608..f3da978 100644 --- a/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py +++ b/fanficdownloader/adapters/adapter_merlinficdtwinscouk.py @@ -268,6 +268,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_midnightwhispersca.py b/fanficdownloader/adapters/adapter_midnightwhispersca.py index 2a9a8f6..5f8c567 100644 --- a/fanficdownloader/adapters/adapter_midnightwhispersca.py +++ b/fanficdownloader/adapters/adapter_midnightwhispersca.py @@ -260,6 +260,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_mugglenetcom.py b/fanficdownloader/adapters/adapter_mugglenetcom.py index dd17128..8962ca0 100644 --- a/fanficdownloader/adapters/adapter_mugglenetcom.py +++ b/fanficdownloader/adapters/adapter_mugglenetcom.py @@ -306,6 +306,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_nationallibrarynet.py b/fanficdownloader/adapters/adapter_nationallibrarynet.py index 00303eb..b525532 100644 --- a/fanficdownloader/adapters/adapter_nationallibrarynet.py +++ b/fanficdownloader/adapters/adapter_nationallibrarynet.py @@ -163,6 +163,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter): if 'Series' in label: self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:]) + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href']) asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId'))) diff --git a/fanficdownloader/adapters/adapter_ncisficcom.py b/fanficdownloader/adapters/adapter_ncisficcom.py index 744ef4f..8deca35 100644 --- a/fanficdownloader/adapters/adapter_ncisficcom.py +++ b/fanficdownloader/adapters/adapter_ncisficcom.py @@ -164,6 +164,7 @@ class NCISFicComAdapter(BaseSiteAdapter): if 'Series' in label: if "No Series" not in value.nextSibling.string: self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:]) + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href']) asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl'))) story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId'))) diff --git a/fanficdownloader/adapters/adapter_ncisfictionnet.py b/fanficdownloader/adapters/adapter_ncisfictionnet.py index 2f64fd9..363fcb2 100644 --- a/fanficdownloader/adapters/adapter_ncisfictionnet.py +++ b/fanficdownloader/adapters/adapter_ncisfictionnet.py @@ -193,7 +193,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter): series_name = a.find('a').string i = a.text.split('#')[1] self.setSeries(series_name, i) - + self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href']) except: # I find it hard to care if the series parsing fails pass diff --git a/fanficdownloader/adapters/adapter_nfacommunitycom.py b/fanficdownloader/adapters/adapter_nfacommunitycom.py index 8c4a275..44fc5c4 100644 --- a/fanficdownloader/adapters/adapter_nfacommunitycom.py +++ b/fanficdownloader/adapters/adapter_nfacommunitycom.py @@ -264,6 +264,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py index 736f992..62863aa 100644 --- a/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py @@ -244,6 +244,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_pommedesangcom.py b/fanficdownloader/adapters/adapter_pommedesangcom.py index 762e75c..33e4192 100644 --- a/fanficdownloader/adapters/adapter_pommedesangcom.py +++ b/fanficdownloader/adapters/adapter_pommedesangcom.py @@ -275,6 +275,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']: self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py index 0bbbf5d..107a527 100644 --- a/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py +++ b/fanficdownloader/adapters/adapter_ponyfictionarchivenet.py @@ -225,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py index a95c620..7385a88 100644 --- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -182,6 +182,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_pretendercentrecom.py b/fanficdownloader/adapters/adapter_pretendercentrecom.py index c5c8a5d..d87ae3f 100644 --- a/fanficdownloader/adapters/adapter_pretendercentrecom.py +++ b/fanficdownloader/adapters/adapter_pretendercentrecom.py @@ -228,6 +228,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_prisonbreakficnet.py b/fanficdownloader/adapters/adapter_prisonbreakficnet.py index e125108..b0c33de 100644 --- a/fanficdownloader/adapters/adapter_prisonbreakficnet.py +++ b/fanficdownloader/adapters/adapter_prisonbreakficnet.py @@ -193,6 +193,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_psychficcom.py b/fanficdownloader/adapters/adapter_psychficcom.py index 0f1d06a..015248a 100644 --- a/fanficdownloader/adapters/adapter_psychficcom.py +++ b/fanficdownloader/adapters/adapter_psychficcom.py @@ -223,6 +223,7 @@ class PsychFicComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_qafficcom.py b/fanficdownloader/adapters/adapter_qafficcom.py index d6b5527..f6bfae0 100644 --- a/fanficdownloader/adapters/adapter_qafficcom.py +++ b/fanficdownloader/adapters/adapter_qafficcom.py @@ -236,6 +236,7 @@ class QafFicComAdapter(BaseSiteAdapter): name=seriessoup.find('div', {'id' : 'pagetitle'}) name.find('a').extract() self.setSeries(name.text.split(' by[')[0], i) + self.story.setMetadata('seriesUrl',series_url) i=0 break i+=1 diff --git a/fanficdownloader/adapters/adapter_samdeanarchivenu.py b/fanficdownloader/adapters/adapter_samdeanarchivenu.py index 55dcf1c..c4374a8 100644 --- a/fanficdownloader/adapters/adapter_samdeanarchivenu.py +++ b/fanficdownloader/adapters/adapter_samdeanarchivenu.py @@ -207,6 +207,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py index fc04e89..d54b2f4 100644 --- a/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py +++ b/fanficdownloader/adapters/adapter_scarvesandcoffeenet.py @@ -222,6 +222,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py index 6852eb8..b5d587f 100644 --- a/fanficdownloader/adapters/adapter_sg1heliopoliscom.py +++ b/fanficdownloader/adapters/adapter_sg1heliopoliscom.py @@ -233,6 +233,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_sinfuldesireorg.py b/fanficdownloader/adapters/adapter_sinfuldesireorg.py index 372196c..89c076c 100644 --- a/fanficdownloader/adapters/adapter_sinfuldesireorg.py +++ b/fanficdownloader/adapters/adapter_sinfuldesireorg.py @@ -226,6 +226,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_siyecouk.py b/fanficdownloader/adapters/adapter_siyecouk.py index 5f9af64..83d5391 100644 --- a/fanficdownloader/adapters/adapter_siyecouk.py +++ b/fanficdownloader/adapters/adapter_siyecouk.py @@ -215,6 +215,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_squidgeorgpeja.py b/fanficdownloader/adapters/adapter_squidgeorgpeja.py index d4a39aa..ecc2472 100644 --- a/fanficdownloader/adapters/adapter_squidgeorgpeja.py +++ b/fanficdownloader/adapters/adapter_squidgeorgpeja.py @@ -212,6 +212,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_stargateatlantisorg.py b/fanficdownloader/adapters/adapter_stargateatlantisorg.py index f8c52ba..c52e433 100644 --- a/fanficdownloader/adapters/adapter_stargateatlantisorg.py +++ b/fanficdownloader/adapters/adapter_stargateatlantisorg.py @@ -204,6 +204,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_svufictioncom.py b/fanficdownloader/adapters/adapter_svufictioncom.py index 2efd863..a2569fe 100644 --- a/fanficdownloader/adapters/adapter_svufictioncom.py +++ b/fanficdownloader/adapters/adapter_svufictioncom.py @@ -249,6 +249,7 @@ class SVUFictionComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py index b3f7a07..50ef360 100644 --- a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py +++ b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py @@ -220,6 +220,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index 72e1c7a..2a38c93 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -101,8 +101,10 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" } self.story.setMetadata('language',langs[idnum%len(langs)]) self.setSeries('The Great Test',idnum) + self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1') if idnum == 0: self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum) + self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0') self.story.setMetadata('rating','Tweenie') diff --git a/fanficdownloader/adapters/adapter_thealphagatecom.py b/fanficdownloader/adapters/adapter_thealphagatecom.py index ff6ee8a..64f3aed 100644 --- a/fanficdownloader/adapters/adapter_thealphagatecom.py +++ b/fanficdownloader/adapters/adapter_thealphagatecom.py @@ -189,6 +189,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_thehookupzonenet.py b/fanficdownloader/adapters/adapter_thehookupzonenet.py index a43ecb1..1a304ff 100644 --- a/fanficdownloader/adapters/adapter_thehookupzonenet.py +++ b/fanficdownloader/adapters/adapter_thehookupzonenet.py @@ -283,6 +283,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py index fc5d48e..8fbf1d5 100644 --- a/fanficdownloader/adapters/adapter_thequidditchpitchorg.py +++ b/fanficdownloader/adapters/adapter_thequidditchpitchorg.py @@ -263,6 +263,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py index 7e9b579..3a97c95 100644 --- a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py +++ b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py @@ -227,6 +227,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py index 3f83bd4..994bb62 100644 --- a/fanficdownloader/adapters/adapter_tthfanficorg.py +++ b/fanficdownloader/adapters/adapter_tthfanficorg.py @@ -255,9 +255,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter): pseries.text) if m: self.setSeries(m.group('series'),m.group('num')) - - return - + self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href']) def getChapterText(self, url): logger.debug('Getting chapter text from: %s' % url) diff --git a/fanficdownloader/adapters/adapter_twilightarchivescom.py b/fanficdownloader/adapters/adapter_twilightarchivescom.py index 3f2eef7..420678c 100644 --- a/fanficdownloader/adapters/adapter_twilightarchivescom.py +++ b/fanficdownloader/adapters/adapter_twilightarchivescom.py @@ -163,6 +163,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('/read/'+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index 8f46aa3..12c9431 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -217,6 +217,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py index cf417ec..f463f10 100644 --- a/fanficdownloader/adapters/adapter_twiwritenet.py +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -246,6 +246,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_walkingtheplankorg.py b/fanficdownloader/adapters/adapter_walkingtheplankorg.py index 53357da..c439a91 100644 --- a/fanficdownloader/adapters/adapter_walkingtheplankorg.py +++ b/fanficdownloader/adapters/adapter_walkingtheplankorg.py @@ -206,6 +206,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_whoficcom.py b/fanficdownloader/adapters/adapter_whoficcom.py index 6f396f7..a0a8ca6 100644 --- a/fanficdownloader/adapters/adapter_whoficcom.py +++ b/fanficdownloader/adapters/adapter_whoficcom.py @@ -204,6 +204,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_wizardtalesnet.py b/fanficdownloader/adapters/adapter_wizardtalesnet.py index b00df83..7f16398 100644 --- a/fanficdownloader/adapters/adapter_wizardtalesnet.py +++ b/fanficdownloader/adapters/adapter_wizardtalesnet.py @@ -277,6 +277,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_wolverineandroguecom.py b/fanficdownloader/adapters/adapter_wolverineandroguecom.py index 24a1bd0..9bf000a 100644 --- a/fanficdownloader/adapters/adapter_wolverineandroguecom.py +++ b/fanficdownloader/adapters/adapter_wolverineandroguecom.py @@ -194,6 +194,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter): if 'contact.php' not in a['href'] and 'index' not in a['href']: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/adapters/adapter_wraithbaitcom.py b/fanficdownloader/adapters/adapter_wraithbaitcom.py index f827d7e..0254c33 100644 --- a/fanficdownloader/adapters/adapter_wraithbaitcom.py +++ b/fanficdownloader/adapters/adapter_wraithbaitcom.py @@ -196,6 +196,7 @@ class WraithBaitComAdapter(BaseSiteAdapter): for a in storyas: if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')): self.setSeries(series_name, i) + self.story.setMetadata('seriesUrl',series_url) break i+=1 diff --git a/fanficdownloader/configurable.py b/fanficdownloader/configurable.py index 261c7a3..4b18c87 100644 --- a/fanficdownloader/configurable.py +++ b/fanficdownloader/configurable.py @@ -50,6 +50,7 @@ class Configuration(ConfigParser.SafeConfigParser): 'characters', 'ships', 'series', + 'seriesUrl', 'status', 'datePublished', 'dateUpdated', @@ -75,6 +76,7 @@ class Configuration(ConfigParser.SafeConfigParser): 'langcode', 'output_css', 'authorHTML', + 'seriesHTML', 'lastupdate' ] diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 617ce09..be9683e 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -352,7 +352,7 @@ class Story(Configurable): allmetadata = {} # special handling for authors/authorUrls - authlinkhtml="%s" + linkhtml="%s" if self.isList('author'): # more than one author, assume multiple authorUrl too. htmllist=[] for i, v in enumerate(self.getList('author')): @@ -366,12 +366,14 @@ class Story(Configurable): aurl=removeAllEntities(aurl) auth=removeAllEntities(auth) - htmllist.append(authlinkhtml%(aurl,auth)) + htmllist.append(linkhtml%('author',aurl,auth)) self.setMetadata('authorHTML',', '.join(htmllist)) else: - self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements), - self.getMetadata('author', removeallentities, doreplacements))) + self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements), + self.getMetadata('author', removeallentities, doreplacements))) + self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements), + self.getMetadata('series', removeallentities, doreplacements))) for k in self.getValidMetaList(): if self.isList(k) and keeplists: allmetadata[k] = self.getList(k, removeallentities, doreplacements)