diff --git a/app.yaml b/app.yaml index cedb49d..6a182e1 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-4-15 +version: 4-4-16 runtime: python27 api_version: 1 threadsafe: true diff --git a/defaults.ini b/defaults.ini index cb3ee5e..00c4903 100644 --- a/defaults.ini +++ b/defaults.ini @@ -425,6 +425,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif ## personal.ini, not defaults.ini. #is_adult:true +[samdean.archive.nu] + +[stargate-atlantis.org] [thehexfiles.net] @@ -447,6 +450,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif ## this should go in your personal.ini, not defaults.ini. #is_adult:true +[www.thealphagate.com] + [www.archiveofourown.org] ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, @@ -461,6 +466,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #username:YourName #password:yourpassword +[www.destinysgateway.com] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. In commandline version, +## this should go in your personal.ini, not defaults.ini. +#is_adult:true + [www.dokuga.com] [www.fanfiction.net] @@ -545,6 +556,8 @@ extratags: ## cover image. This lets you exclude them. cover_exclusion_regexp:/stories/999/images/.*?_trophy.png +[www.ncisfiction.com] + [www.phoenixsong.net] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In @@ -637,6 +650,12 @@ collect_series: false ## this should go in your personal.ini, not defaults.ini. #is_adult:true +[www.yourfanfiction.com] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. In commandline version, +## this should go in your personal.ini, not defaults.ini. +#is_adult:true + [overrides] ## It may sometimes be useful to override all of the specific format, ## site and site:format sections in your private configuration. For diff --git a/downloader.py b/downloader.py index abc3781..deb1e67 100644 --- a/downloader.py +++ b/downloader.py @@ -17,7 +17,7 @@ import logging ## XXX cli option for logging level. -logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s") +logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s") import sys, os from os.path import normpath, expanduser, isfile, join diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 020ce82..f2e72ee 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -70,6 +70,12 @@ import adapter_iketernalnet import adapter_onedirectionfanfictioncom import adapter_prisonbreakficnet import adapter_storiesofardacom +import adapter_yourfanfictioncom +import adapter_samdeanarchivenu +import adapter_destinysgatewaycom +import adapter_ncisfictioncom +import adapter_stargateatlantisorg +import adapter_thealphagatecom ## This bit of complexity allows adapters to be added by just adding ## importing. It eliminates the long if/else clauses we used to need diff --git a/fanficdownloader/adapters/adapter_destinysgatewaycom.py b/fanficdownloader/adapters/adapter_destinysgatewaycom.py index 8a88d7d..f4757a9 100644 --- a/fanficdownloader/adapters/adapter_destinysgatewaycom.py +++ b/fanficdownloader/adapters/adapter_destinysgatewaycom.py @@ -96,11 +96,27 @@ class DestinysGatewayComAdapter(BaseSiteAdapter): else: raise e - # The actual text that is used to announce you need to be an - # adult varies from site to site. Again, print data before - # the title search to troubleshoot. - if "This rating has graphic sexual scenes" in data: - raise exceptions.AdultCheckRequired(self.url) + m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data) + if m != None: + if self.is_adult or self.getConfig("is_adult"): + # We tried the default and still got a warning, so + # let's pull the warning number from the 'continue' + # link and reload data. + addurl = m.group(1) + # correct stupid & error in url. + addurl = addurl.replace("&","&") + url = self.url+'&index=1'+addurl + logging.debug("URL 2nd try: "+url) + + try: + data = self._fetchUrl(url) + except urllib2.HTTPError, e: + if e.code == 404: + raise exceptions.StoryDoesNotExist(self.url) + else: + raise e + else: + raise exceptions.AdultCheckRequired(self.url) if "Access denied. This story has not been validated by the adminstrators of this site." in data: raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") diff --git a/fanficdownloader/adapters/adapter_samdeanarchivenu.py b/fanficdownloader/adapters/adapter_samdeanarchivenu.py index 51a20ae..38c6f27 100644 --- a/fanficdownloader/adapters/adapter_samdeanarchivenu.py +++ b/fanficdownloader/adapters/adapter_samdeanarchivenu.py @@ -69,11 +69,15 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter): # The site domain. Does have www here, if it uses it. return 'samdean.archive.nu' + @classmethod + def getAcceptDomains(cls): + return ['www.samdean.archive.nu','samdean.archive.nu'] + def getSiteExampleURLs(self): return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234" def getSiteURLPattern(self): - return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" + return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$" ## Getting the chapter list and the meta data, plus 'is adult' checking. def extractChapterUrlsAndMetadata(self): diff --git a/fanficdownloader/adapters/adapter_yourfanfictioncom.py b/fanficdownloader/adapters/adapter_yourfanfictioncom.py index 71fe0fb..6b618d5 100644 --- a/fanficdownloader/adapters/adapter_yourfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_yourfanfictioncom.py @@ -57,7 +57,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter): # The date format will vary from site to site. # http://docs.python.org/library/datetime.html#strftime-strptime-behavior - self.dateformat = "%d/%b/%y" + self.dateformat = "%d %b %Y" @staticmethod # must be @staticmethod, don't remove it. def getSiteDomain(): diff --git a/index.html b/index.html index c3ce58d..c0a10ee 100644 --- a/index.html +++ b/index.html @@ -56,7 +56,12 @@

New Sites

- New sites onedirectionfanfiction.com, www.prisonbreakfic.net and www.storiesofarda.com added. Thanks, Ida. + + New sites samdean.archive.nu, www.yourfanfiction.com, + www.destinysgateway.com, www.thealphagate.com, + stargate-atlantis.org and www.ncisfiction.com added. + That's 51 different supported sites now. Thanks, Ida! +

Questions? Check out our @@ -66,7 +71,7 @@ If you have any problems with this application, please report them in the FanFictionDownLoader Google Group. The - Previous Version is also available for you to use if necessary. + Previous Version is also available for you to use if necessary.

{{ error_message }} @@ -384,6 +389,38 @@ Use the URL of the story's chapter list, such as
http://www.storiesofarda.com/chapterlistview.asp?SID=7000 +
samdean.archive.nu
+
+ Use the URL of the story's first chapter, such as +
http://samdean.archive.nu/viewstory.php?sid=1234 +
+
www.yourfanfiction.com
+
+ Use the URL of the story's first chapter, such as +
http://www.yourfanfiction.com/viewstory.php?sid=1234 +
+
www.destinysgateway.com
+
+ Use the URL of the story's first chapter, such as +
http://www.destinysgateway.com/viewstory.php?sid=1234 +
+
www.thealphagate.com
+
+ Use the URL of the story's first chapter, such as +
http://www.thealphagate.com/viewstory.php?sid=1234 +
+
stargate-atlantis.org
+
+ Use the URL of the story's first chapter, such as +
http://stargate-atlantis.org/fanfics/viewstory.php?sid=1234 +
+
www.ncisfiction.com
+
+ Use the URL of the story's first chapter, such as +
http://www.ncisfiction.com/story.php?stid=1234 + Or use the URL of the story's chapter list, such as +
http://www.ncisfiction.com/chapters.php?stid=1234 +

A few additional things to know, which will make your life substantially easier: diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 557583a..7885303 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -411,6 +411,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif ## personal.ini, not defaults.ini. #is_adult:true +[samdean.archive.nu] + +[stargate-atlantis.org] + [thehexfiles.net] [thequidditchpitch.org] @@ -432,6 +436,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif ## this should go in your personal.ini, not defaults.ini. #is_adult:true +[www.thealphagate.com] + [www.archiveofourown.org] ## Some sites do not require a login, but do require the user to ## confirm they are adult for adult content. In commandline version, @@ -446,6 +452,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif #username:YourName #password:yourpassword +[www.destinysgateway.com] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. In commandline version, +## this should go in your personal.ini, not defaults.ini. +#is_adult:true + [www.dokuga.com] [www.fanfiction.net] @@ -527,6 +539,8 @@ extratags: ## cover image. This lets you exclude them. cover_exclusion_regexp:/stories/999/images/.*?_trophy.png +[www.ncisfiction.com] + [www.phoenixsong.net] ## Some sites require login (or login for some rated stories) The ## program can prompt you, or you can save it in config. In @@ -619,6 +633,12 @@ collect_series: false ## this should go in your personal.ini, not defaults.ini. #is_adult:true +[www.yourfanfiction.com] +## Some sites do not require a login, but do require the user to +## confirm they are adult for adult content. In commandline version, +## this should go in your personal.ini, not defaults.ini. +#is_adult:true + [overrides] ## It may sometimes be useful to override all of the specific format, ## site and site:format sections in your private configuration. For