diff --git a/public/index.html b/public/index.html
index f3ab4c6..b85ffc7 100644
--- a/public/index.html
+++ b/public/index.html
@@ -473,7 +473,7 @@ lying internet
-
+
diff --git a/public/js/main.js b/public/js/main.js
index 0e5dc95..42c5841 100644
--- a/public/js/main.js
+++ b/public/js/main.js
@@ -12,87 +12,10 @@ var KWS = function(){
keywordsToQuery: [],
keywordsToQueryIndex: 0,
numOfInitialKeywords: 0,
- services:{
- "google":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
- "google news":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
- "google shopping":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
- "google books":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
- "youtube":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
- "google videos":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
- "google images":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
- "yahoo":
- "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
- "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
- "ebay":
- "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
- "amazon":
- "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
- "twitter":
- "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
- "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
- "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
- "google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
- "google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
- "google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
- "google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
- // "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
- },
- /**
- * Get the service url based on options set in the dom.
- * @return {String} A jsonp url for search suggestions with query missing from the end.
- */
- getUrl :function(service, options){
- // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
- // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
- // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
- // https://developers.google.com/custom-search/docs/ref_languages
- return _.template(this.services[(service||this.options.service)])(options||this.options);
- },
-
- /** Parse response per service **/
- parseServiceResponse: function(res, service){
- // Each take a json response tand return a keyword array
- RESPONSE_TEMPLATES = {
- // opensearch default
- "default": function (res) {
- return res[1];
- },
- "yahoo": function (res) {
- return _.map(res.gossip.results, 'key');
- },
- "ebay": function (res) {
- return res.res ? res.res.sug : [];
- },
- "twitter": function (res) {
- return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
- },
- "baidu": function (res) {
- return res.s;
- },
- "yandex": function(res){
- return _.map(res[1], function(r){
- return typeof r === 'string' ? r : r[1];
- });
- },
- "linkedin": function(res){
- return _.map(res.resultList,'displayName');
- },
- "google play": function(res){return _.map(res,'s')},
- "google play apps": function(res){return _.map(res,'s')},
- "google play movies": function(res){return _.map(res,'s')},
- "google play books": function(res){return _.map(res,'s')},
- };
- var parser = RESPONSE_TEMPLATES[(service||this.options.service)] || RESPONSE_TEMPLATES["default"];
- return parser(res);
- },
+ services:suggestions.services,
+ getUrl :suggestions.getUrl,
+ parseServiceResponse: suggestions.parseServiceResponse,
toggleWork: function(){
if (this.doWork === false)
@@ -119,7 +42,7 @@ var KWS = function(){
// get queries from the input
var ks = $('#input').val().split("\n");
- this.keywordsToQuery=ks.map(this.CleanVal);
+ this.keywordsToQuery=_.map(ks,this.CleanVal);
// add variations of the initial terms
@@ -192,7 +115,7 @@ var KWS = function(){
},
addResultsToQueue: function(retList, search){
- retList=retList.map(this.CleanVal);
+ retList=_.map(retList,this.CleanVal);
// add each result to list first before permutations
for (var j = 0; j < retList.length; j++) {
@@ -209,6 +132,7 @@ var KWS = function(){
permuteResultsToQueue: function(retList, search){
var chr, currentx, currentKw;
var self = this;
+ var options = this.getOptions()
this.hashMapInputs[search] = true;
@@ -224,14 +148,14 @@ var KWS = function(){
return s+' '+suffix;
}
// clean
- retList=retList.map(this.CleanVal);
+ retList=_.map(retList,this.CleanVal);
// get permutations
var newInputs = retList.reduce(function(result, keyword){
return _.concat(
result,
- self.options.prefixes.map(addPrefix.bind(self,keyword)),
- self.options.suffixes.map(addSuffix.bind(self,keyword))
+ _.map(options.prefixes,addPrefix.bind(self,keyword)),
+ _.map(options.suffixes,addSuffix.bind(self,keyword))
);
}, []);
@@ -246,7 +170,7 @@ var KWS = function(){
displayResults: function(retList, search, dontDisplay, url,data){
var rows=[];
- retList=retList.map(this.CleanVal);
+ retList=_.map(retList,this.CleanVal);
for (var i = 0; i < retList.length; i++) {
var cleanKw = retList[i];
diff --git a/public/js/suggestions.js b/public/js/suggestions.js
new file mode 100644
index 0000000..c443cac
--- /dev/null
+++ b/public/js/suggestions.js
@@ -0,0 +1,100 @@
+var suggestions = function(){
+ return {
+ defaultOptions:{
+ deferTableUpdatesAtRows: 5000,
+ keepRunning: false,
+ maxQueueDisplay: 5000,
+ country: "",
+ filterNegative: "",
+ filterPositive: "",
+ lang: "",
+ prefixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "how", "which", "why", "where", "who", "when", "are", "what"],
+ rateLimit: 750,
+ service: "google",
+ suffixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "like", "for", "without", "with", "versus", "vs", "to", "near", "except", "has"]
+ },
+
+ services:{
+ "google":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
+ "google news":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
+ "google shopping":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
+ "google books":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
+ "youtube":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
+ "google videos":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
+ "google images":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
+ "yahoo":
+ "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
+ "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
+ "ebay":
+ "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
+ "amazon":
+ "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
+ "twitter":
+ "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
+ "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
+ "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
+ "google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
+ "google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
+ "google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
+ "google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
+ // "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
+ },
+ /**
+ * Get the service url based on options set in the dom.
+ * @return {String} A jsonp url for search suggestions with query missing from the end.
+ */
+ getUrl :function(service, options){
+ // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
+ // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
+ // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
+ // https://developers.google.com/custom-search/docs/ref_languages
+ options = _.defaults({},this.defaultOptions, this.getOptions(), options)
+ return _.template(this.services[(service||this.options.service)])(options);
+ },
+
+
+ /** Parse response per service **/
+ parseServiceResponse: function(res, service){
+ // Each take a json response tand return a keyword array
+ RESPONSE_TEMPLATES = {
+ // opensearch default
+ "default": function (res) {
+ return res[1];
+ },
+ "yahoo": function (res) {
+ return _.map(res.gossip.results, 'key');
+ },
+ "ebay": function (res) {
+ return res.res ? res.res.sug : [];
+ },
+ "twitter": function (res) {
+ return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
+ },
+ "baidu": function (res) {
+ return res.s;
+ },
+ "yandex": function(res){
+ return _.map(res[1], function(r){
+ return typeof r === 'string' ? r : r[1];
+ });
+ },
+ "linkedin": function(res){
+ return _.map(res.resultList,'displayName');
+ },
+ "google play": function(res){return _.map(res,'s')},
+ "google play apps": function(res){return _.map(res,'s')},
+ "google play movies": function(res){return _.map(res,'s')},
+ "google play books": function(res){return _.map(res,'s')},
+ };
+ var parser = RESPONSE_TEMPLATES[(service||this.options.service)] || RESPONSE_TEMPLATES["default"];
+ return parser(res);
+ }
+ }
+}()
diff --git a/public/js/test.js b/public/js/test.js
index 8b989fe..670d12f 100644
--- a/public/js/test.js
+++ b/public/js/test.js
@@ -16,35 +16,15 @@ describe('permute', function(){
it('should correctly permute results and add to queue', function() {
var retList = ['a','b','c d',' a longer one'];
KWS.permuteResultsToQueue(retList,'test');
- assert(KWS.keywordsToQuery.length>0);
- assert(KWS.hashMapInputs['b']);
- assert(KWS.hashMapInputs['test']);
+ expect(KWS.keywordsToQuery).to.have.length.above(0)
+ // assert(KWS.hashMapInputs['b']);
+ expect(KWS.hashMapInputs).to.include.key('test');
// var options = KWS.getOptions();
// var expectedLength = options.prefixes.length*retList.length+options.suffixes.length*retList.length
// assert(KWS.keywordsToQuery.length==expectedLength,''+expectedLength+'!='+KWS.keywordsToQuery.length);
});
});
-// TODO use localForage or most popular promise wrapper for indexedDB then test
-// describe('exportDB', function(){
-// });
-
-describe('db', function() {
- it('should correctly set up db', function(done) {
- var dbReq=KWS.setUpDb();
- var oldonsuccess=dbReq.onsuccess;
- dbReq.onsuccess=function(event){
- // override onld onsucess because it doesn't use promises
- var db=oldonsuccess.bind(dbReq)(event);
- assert(db);
- assert(db.objectStoreNames.contains('suggestions'),'db should contain suggestons store');
- var indexNames =db.transaction('suggestions','readonly').objectStore('suggestions').indexNames;
- assert(indexNames.contains('keyword','search'));
- done();
- };
- });
-});
-
describe('extractDomain', function(){
@@ -66,11 +46,10 @@ describe('responses', function(){
if (testData.responses.hasOwnProperty(service)) {
it('should correctly parse test data for service='+service, function() {
var res = testData.responses[service];
- var options = {service:service};
- var parsedRes = KWS.parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,service);
- assert.typeOf(parsedRes,'array');
- expect(parsedRes).to.have.length.above(1);
+ expect(parsedRes).to.be.an('array')
+ .that.has.length.above(1);
parsedRes.map(r => assert.typeOf(r, 'string'));
});
}
@@ -101,8 +80,7 @@ describe('services', function() {
services.forEach(function(service){
searches.forEach(function(search){
it('should correctly get and parse data'+'. Service="'+service+'" search="'+search+'"', function(done) {
- var options = {service:service};
- var url = KWS.getUrl(options)+search;
+ var url = KWS.getUrl(service)+search;
return $.ajax({
url: url,
async: false,
@@ -111,7 +89,7 @@ describe('services', function() {
success: function (res, statusText, jqXHR) {
assert(statusText=="success");
assert(res!==[]);
- var parsedRes = KWS.parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,service);
assert.typeOf(parsedRes,'array');
parsedRes.map(r => assert.typeOf(r, 'string'));
done();
@@ -129,8 +107,7 @@ describe('services', function() {
services.forEach(function(service){
searchesDifficult.forEach(function(search){
it('should get and parse difficult data'+'. Service="'+service+'" search="'+search+'"', function(done) {
- var options = {service:service};
- var url = KWS.getUrl(options)+search;
+ var url = KWS.getUrl(service)+search;
return $.ajax({
url: url,
async: false,
@@ -140,7 +117,7 @@ describe('services', function() {
assert(statusText=="success");
assert(res!==[]);
- var parsedRes = KWS.parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,service);
assert.typeOf(parsedRes,'array');
parsedRes.map(r => assert.typeOf(r, 'string'));
return done();
diff --git a/public/test.html b/public/test.html
index c6b4782..e1584a0 100644
--- a/public/test.html
+++ b/public/test.html
@@ -25,6 +25,7 @@
+