From aa58f1c1eedf8f52193231469ba5049503b8dffd Mon Sep 17 00:00:00 2001
From: Jim Miller <retiefjimm@gmail.com>
Date: Sat, 28 May 2011 16:09:34 -0500
Subject: [PATCH] Make much more forgiving of inexact URLs--that's most of our
 errors now.

---
 fanficdownloader/adapters/__init__.py         | 38 +++++++++++++++----
 .../adapters/adapter_fanfictionnet.py         |  2 +-
 .../adapters/adapter_fictionpresscom.py       |  2 +-
 fanficdownloader/adapters/adapter_test1.py    |  2 +-
 4 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py
index 8213de9..0800ca4 100644
--- a/fanficdownloader/adapters/__init__.py
+++ b/fanficdownloader/adapters/__init__.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import os, sys, glob
+import os, re, sys, glob
 from os.path import dirname, basename, normpath
 import logging
 import urlparse as up
@@ -31,15 +31,39 @@ import fanficdownloader.exceptions as exceptions
 __class_list = []
 
 def getAdapter(config,url):
-    parsedUrl = up.urlparse(url)
-    logging.debug("site:"+parsedUrl.netloc)
-    for cls in __class_list:
-        if cls.matchesSite(parsedUrl.netloc):
-            adapter = cls(config,url) # raises InvalidStoryURL
-            return adapter
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls
+
 ## Automatically import each adapter_*.py file.
 ## Each implement getClass() to their class
 
diff --git a/fanficdownloader/adapters/adapter_fanfictionnet.py b/fanficdownloader/adapters/adapter_fanfictionnet.py
index 8cd0865..ab68784 100644
--- a/fanficdownloader/adapters/adapter_fanfictionnet.py
+++ b/fanficdownloader/adapters/adapter_fanfictionnet.py
@@ -51,7 +51,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+/\d+(/|/[a-zA-Z0-9_]+)?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?$"
 
     def extractChapterUrlsAndMetadata(self):
 
diff --git a/fanficdownloader/adapters/adapter_fictionpresscom.py b/fanficdownloader/adapters/adapter_fictionpresscom.py
index 7902296..19cfa45 100644
--- a/fanficdownloader/adapters/adapter_fictionpresscom.py
+++ b/fanficdownloader/adapters/adapter_fictionpresscom.py
@@ -47,7 +47,7 @@ class FictionPressComSiteAdapter(BaseSiteAdapter):
         return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://www\.fictionpress\.com/s/\d+/\d+(/|/[a-zA-Z0-9_]+)?$"
+        return r"http://www\.fictionpress\.com/s/\d+(/\d)?(/|/[a-zA-Z0-9_-]+)?$"
 
     def extractChapterUrlsAndMetadata(self):
 
diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py
index f3d3603..7b7e94c 100644
--- a/fanficdownloader/adapters/adapter_test1.py
+++ b/fanficdownloader/adapters/adapter_test1.py
@@ -42,7 +42,7 @@ class TestSiteAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"?sid=1234"
 
     def getSiteURLPattern(self):
-        return BaseSiteAdapter.getSiteURLPattern(self)+'\?sid=\d+$'
+        return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
     def extractChapterUrlsAndMetadata(self):