From ad97240d379fa1ed6c115ebf27a3c1123502007b Mon Sep 17 00:00:00 2001
From: Is Isilon <wassname@wassname.org>
Date: Tue, 1 Mar 2016 17:04:55 +0800
Subject: [PATCH] Autogen service options

---
 package.json          |   9 +++-
 public/index.html     |  69 ++++++++++++++-----------
 public/js/main.js     | 117 +++++++++++++++++++++++++-----------------
 public/js/test.js     |   3 +-
 public/js/testData.js |  19 +++++++
 public/test.html      |   2 +-
 6 files changed, 136 insertions(+), 83 deletions(-)

diff --git a/package.json b/package.json
index 1ea7d47..ca1922d 100644
--- a/package.json
+++ b/package.json
@@ -8,15 +8,20 @@
     "test": "public/test.html",
     "build-css": "lessc ./src/less/main.less > public/css/style.css",
     "build": "npm run build-css",
-    "start": "node server.js"
+    "start": "node server.js",
+    "debug": "NODE_ENV=dev DEBUG=* nodemon server",
+    "production": "NODE_ENV=production npm start"
   },
   "author": "blackmia,wassname",
   "license": "MIT",
   "devDependencies": {
     "chai": "^3.5.0",
     "cors": "^2.7.1",
+    "debug": "^2.2.0",
     "express": "^4.13.4",
     "lessc": "latest",
-    "mocha": "^2.4.5"
+    "mocha": "^2.4.5",
+    "morgan": "^1.6.1",
+    "winston": "^2.1.1"
   }
 }
diff --git a/public/index.html b/public/index.html
index f77934d..7383976 100644
--- a/public/index.html
+++ b/public/index.html
@@ -72,7 +72,7 @@ lying internet</textarea>
                     <div class="form-group">
                       <label for="service">Service:</label>
                       <select class="form-control" name="service" id="service" >
-                        <option>google</option>
+                        <!-- <option>google</option>
                         <option>twitter</option>
                         <option>yahoo</option>
                         <option>bing</option>
@@ -86,8 +86,32 @@ lying internet</textarea>
                         <option>google shopping</option>
                         <option>yandex</option>
                         <option>baidu</option>
+                        <option>linkedin</option> -->
+                        <!-- <option>yelp</option> -->
                       </select>
                     </div>
+
+
+                    <div class="form-group">
+                      <label for="prefixes">Prefixes</label>
+                      <textarea type="text" title="These values are added before words to prompt the search suggestions" class="form-control" name="prefixes" id="prefixes"  > ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,how,which,why,where,who,when,are,what</textarea>
+                    </div>
+                    <div class="form-group">
+                      <label for="suffixes">Suffixes:</label>
+                      <textarea type="text" rows="2" title="These values are added after words to prompt the search suggestions" class="form-control" id="suffixes"> ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,like,for,without,with,versus,vs,to,near,except,has</textarea>
+                    </div>
+
+                    <div class="form-group" style="display: none">
+                      <label for="rate-limit">Rate limit (for experts):</label>
+                      <input type="text"  class="form-control" id="rate-limit" value="750" name="rate-limit" title="Value between searches in milliseconds only change if you know what you are doing. This could get your ip banned, or place an unfair burden on the suggest servers." disabled>
+                    </div>
+
+                    <div class="checkbox">
+                      <label for="keep-running">
+                          <input type="checkbox" id="keep-running" name="keep-running" title="Keep running forever"></input>Continuous running:
+                      </label>
+                    </div>
+
                     <div class="form-group">
                       <label for="country">Country code (google):</label>
                       <!-- ISO-3366-1: Alpha-2 Codes -->
@@ -389,34 +413,6 @@ lying internet</textarea>
                       </select>
                     </div>
 
-                    <div class="form-group">
-                      <label for="prefixes">Prefixes</label>
-                      <textarea type="text" title="These values are added before words to prompt the search suggestions" class="form-control" name="prefixes" id="prefixes"  > ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,how,which,why,where,who,when,are,what</textarea>
-                    </div>
-                    <div class="form-group">
-                      <label for="suffixes">Suffixes:</label>
-                      <textarea type="text" rows="2" title="These values are added after words to prompt the search suggestions" class="form-control" id="suffixes"> ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,like,for,without,with,versus,vs,to,near,except,has</textarea>
-                    </div>
-
-                    <div class="form-group"  style="display: none">
-                        <label for="filter-positive" >Positive filter:</label>
-                        <textarea disabled id="filter-positive" name="filter-positive" class="form-control" rows="2"
-                            placeholder="Positive Filter"></textarea>
-                    </div>
-                    <div class="form-group"  style="display: none">
-                        <label for="filter-negative">Negative filter:</label>
-                        <textarea disabled id="filter-negative" name="filter-negative" class="form-control" rows="2"placeholder="Negative Filter"></textarea>
-                    </div>
-                    <div class="form-group" style="display: none">
-                      <label for="rate-limit">Rate limit (for experts):</label>
-                      <input type="text"  class="form-control" id="rate-limit" value="750" name="rate-limit" title="Value between searches in milliseconds only change if you know what you are doing. This could get your ip banned, or place an unfair burden on the suggest servers." disabled>
-                    </div>
-
-                    <div class="checkbox">
-                      <label for="keep-running">
-                          <input type="checkbox" id="keep-running" name="keep-running" title="Keep running forever"></input>Continuous running:
-                      </label>
-                    </div>
 
                 </form>
 
@@ -471,11 +467,11 @@ lying internet</textarea>
 
     </div>
 
-    <script src="//cdnjs.cloudflare.com/ajax/libs/datatables/1.10.10/js/jquery.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.js"></script>
     <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" crossorigin="anonymous"></script>
     <script src="//cdn.datatables.net/s/bs/jszip-2.5.0,dt-1.10.10,b-1.1.0,b-colvis-1.1.0,b-html5-1.1.0,b-print-1.1.0,cr-1.3.0,fh-3.1.0,kt-2.1.0,r-2.0.0,se-1.1.0/datatables.min.js" crossorigin="anonymous"></script>
     <script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/4.2.0/lodash.min.js"></script>
-    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
+    <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
     <!-- <script src="//cdnjs.cloudflare.com/ajax/libs/localforage/1.3.3/localforage.js"></script> -->
     <script src="//cdnjs.cloudflare.com/ajax/libs/progress.js/0.1.0/progress.min.js"></script>
 
@@ -488,6 +484,17 @@ lying internet</textarea>
     <a href="https://github.com/wassname/keywordshitter2"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://camo.githubusercontent.com/652c5b9acfaddf3a9c326fa6bde407b87f7be0f4/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f6f72616e67655f6666373630302e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"></a>
     <!-- Fork me on github... Do you get it? Do you! -->
 
+    <!-- Lets see if anyone ever uses this  -->
+    <script>
+      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+      })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+      ga('create', 'UA-51809277-6', 'auto');
+      ga('send', 'pageview');
+
+    </script>
     <script>
     $(document).ready(function () {
         KWS.init();
diff --git a/public/js/main.js b/public/js/main.js
index 21d37b8..fade330 100644
--- a/public/js/main.js
+++ b/public/js/main.js
@@ -13,6 +13,38 @@ var KWS = function(){
         keywordsToQuery: [],
         keywordsToQueryIndex: 0,
         numOfInitialKeywords: 0,
+        services:{
+            "google":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
+            "google news":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
+            "google shopping":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
+            "google books":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
+            "youtube":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
+            "google videos":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
+            "google images":
+            "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
+            "yahoo":
+            "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
+            "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
+            "ebay":
+            "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
+            "amazon":
+            "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
+            "twitter":
+            "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
+            "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
+            "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
+            "google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
+            "google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
+            "google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
+            "google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
+            // "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
+        },
         /**
          * Get the service url based on options set in the dom.
          * @return {String} A jsonp url for search suggestions with query missing from the end.
@@ -22,35 +54,7 @@ var KWS = function(){
             // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
             // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
             // https://developers.google.com/custom-search/docs/ref_languages
-            services={
-                    "google":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
-                    "google news":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
-                    "google shopping":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
-                    "google books":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
-                    "youtube":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
-                    "google videos":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
-                    "google images":
-                    "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
-                    "yahoo":
-                    "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
-                    "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
-                    "ebay":
-                    "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
-                    "amazon":
-                    "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
-                    "twitter":
-                    "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
-                    "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
-                    "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
-                };
-
-            return _.template(services[this.options.service])(this.options);
+            return _.template(this.services[this.options.service])(this.options);
         },
 
 
@@ -78,7 +82,14 @@ var KWS = function(){
                     return _.map(res[1], function(r){
                         return typeof r === 'string' ? r : r[1];
                     });
-                }
+                },
+                "linkedin": function(res){
+                    return _.map(res.resultList,'displayName');
+                },
+                "google play": function(res){return _.map(res,'s')},
+                "google play apps": function(res){return _.map(res,'s')},
+                "google play movies": function(res){return _.map(res,'s')},
+                "google play books": function(res){return _.map(res,'s')},
             };
             var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"];
             return parser(res);
@@ -493,7 +504,7 @@ var KWS = function(){
                         url = self.getUrl()+search;
                         $.ajax({
                             url: url,
-                            jsonp: "jsonp",
+                            // jsonp: "jsonp",
                             dataType: "jsonp",
                             success: function (res, statusText, jqXHR) {
 
@@ -513,10 +524,13 @@ var KWS = function(){
                                 return;
 
                             },
-                            error: function(e){
-                                console.error(e);
+                            error: function(jqXHR,errorText,error){
+                                console.error(errorText,this.url,this,jqXHR,error);
                                 self.queryLock = false;
                                 return;
+                            },
+                            callback: function(){
+                                console.log(this,arguments);
                             }
                         });
                     }
@@ -526,26 +540,23 @@ var KWS = function(){
 
         /** Clean input, may not all be needed **/
         CleanVal: function(input) {
+            // We want to clean search terms but it's not possible to do this perfectly
+            // as differen't search engines strip differen't amounts from the term
+            // so we will keep as much details as possible
 
-            function escapeHtml(unsafe) {
-                return unsafe
-                     .replace(/&/g, "&amp;")
-                     .replace(/</g, "&lt;")
-                     .replace(/>/g, "&gt;")
-                     .replace(/"/g, "&quot;")
-                     .replace(/'/g, "&#039;");
-             }
+            // Search engines are sensitive to whitespace so we do not want to trim
+            // Some return html or escaped html, so we do want to convert to text
 
-            // google returns lowercase
+            // removed escaped html and html tags
+            // e.g. '<b>A&amp;M</b>' => 'A&M'
+            input=$('<div />').html(input).text();
+
+            // I don't know of any search engines sentitive or case so make all lowercase
             input = input.toLowerCase();
-            input = escapeHtml(input);
 
-            // not sure if we want to trim, the search engines are sensitive to
-            // whitespace. (and also last search)
-            // input = input.trim();
-
-            // this removes navigation suggestions
+            // this removes navigation suggestions, perhaps we need to move this to result parser
             if (input.length > 4 && input.substring(0, 4) == "http") input = "";
+
             return input;
         },
 
@@ -711,6 +722,15 @@ var KWS = function(){
 
         init: function(){
             this.setUpDb();
+
+            // add this.servicess to search engine settings
+            for (var service in this.services) {
+                if (this.services.hasOwnProperty(service)) {
+                    $('#service').append('<option>'+service+'</option>')
+                }
+            }
+
+
             this.loadSettings();
             this.options = this.getOptions();
 
@@ -727,6 +747,7 @@ var KWS = function(){
             $('#clear-cache').on('click',this.clearDB.bind(this));
 
 
+
             this.table = $('#outtable').DataTable({
                 pageLength: 25,
                 "lengthMenu": [ 10, 25, 50, 75, 100,800],
diff --git a/public/js/test.js b/public/js/test.js
index e27081a..8b989fe 100644
--- a/public/js/test.js
+++ b/public/js/test.js
@@ -84,7 +84,8 @@ describe('services', function() {
     this.slow(2000);
 
     // data
-    var services = ["google", "google news", "google shopping", "google books", "youtube", "google videos", "google images", "yahoo", "bing", "ebay", "amazon", "twitter", "baidu", "yandex"];
+    // var services = ["google", "google news", "google shopping", "google books", "youtube", "google videos", "google images", "yahoo", "bing", "ebay", "amazon", "twitter", "baidu", "yandex"];
+    var services = Object.keys(KWS.services);
     var searches = ["where"];
     var searchesDifficult = [" * ", "❥"];
 
diff --git a/public/js/testData.js b/public/js/testData.js
index b95e701..4a71c52 100644
--- a/public/js/testData.js
+++ b/public/js/testData.js
@@ -16,6 +16,25 @@ var testData = function () {
             "twitter.com": "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=jQuery111&q=I accidentally&_=11111"
         },
         responses: {
+            "linkedin": {
+                "resultList": [{
+                    "sourceID": "sitefeature",
+                    "displayName": "Where is the Answers feature?",
+                    "subLine": "",
+                    "rank": 0,
+                    "id": "162",
+                    "url": "http://help.linkedin.com/app/answers/global/id/35227/trk/search-typeahead",
+                    "headLine": "<strong>Where<\/strong> is the Answers feature?"
+                }, {
+                    "sourceID": "sitefeature",
+                    "displayName": "Where are the &quot;Settings&quot; and &quot;Sign Out&quot; links on my homepage?",
+                    "subLine": "",
+                    "rank": 1,
+                    "id": "205",
+                    "url": "http://help.linkedin.com/app/answers/global/id/4642/trk/search-typeahead",
+                    "headLine": "<strong>Where<\/strong> are the &quot;Settings&quot; and &quot;Sign Out&quot; links on my homepage?"
+                }]
+            },
             "yahoo": {
                 "gossip": {
                     "qry": "where",
diff --git a/public/test.html b/public/test.html
index dc45486..c6b4782 100644
--- a/public/test.html
+++ b/public/test.html
@@ -21,7 +21,7 @@
   <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" crossorigin="anonymous"></script>
   <script src="//cdn.datatables.net/s/bs/jszip-2.5.0,dt-1.10.10,b-1.1.0,b-colvis-1.1.0,b-html5-1.1.0,b-print-1.1.0,cr-1.3.0,fh-3.1.0,kt-2.1.0,r-2.0.0,se-1.1.0/datatables.min.js" crossorigin="anonymous"></script>
   <script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/4.2.0/lodash.min.js"></script>
-  <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
+  <script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
 
   <script src="//cdnjs.cloudflare.com/ajax/libs/localforage/1.3.3/localforage.js"></script>