diff --git a/.gitignore b/.gitignore index 58c3d86..e1174fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ +notes + # Created by https://www.gitignore.io/api/node,bower,linux ### Node ### diff --git a/css/style.css b/css/style.css index de240d1..e9ea7b4 100644 --- a/css/style.css +++ b/css/style.css @@ -6888,3 +6888,7 @@ label { .pagination { margin: 0px; } +.dataTables_wrapper .dataTables_paginate .paginate_button { + margin: 0px; + padding: 0px; +} diff --git a/index.html b/index.html index f21f4d3..126af31 100644 --- a/index.html +++ b/index.html @@ -4,6 +4,18 @@ + + + + + + + + + + + + @@ -11,7 +23,7 @@ - +
@@ -25,17 +37,18 @@
-
+
-
- Shit Keywords! + + Shit Keywords! - Load cache - Reset - - + Reset +
+
+
+ Load cache + Export cache + Delete cache
- - @@ -63,33 +79,343 @@ lying internet + + + + + +
+
+ + + +
+
+ +
- - + +
- - + +
-
+ -
+ -
+ + +
+
@@ -112,23 +438,26 @@ lying internet
-
+
+
-
+

Results

- - + @@ -152,15 +481,27 @@ lying internet + + + + + + - Fork me on GitHub - + + + + diff --git a/js/main.js b/js/main.js index b5e7ab0..21d37b8 100644 --- a/js/main.js +++ b/js/main.js @@ -1,543 +1,837 @@ -var keywordsToDisplay = []; -var hashMapResults = {}; -var numOfInitialKeywords = 0; -var doWork = false; -var keywordsToQuery = []; -var keywordsToQueryIndex = 0; -var queryLock = false; +var KWS = function(){ + + return { + db:undefined, + table: undefined, + myIp: undefined, + options: {}, + // flags + queryLock: false, + doWork: false, + // keeping track of queue + hashMapInputs: {}, + keywordsToQuery: [], + keywordsToQueryIndex: 0, + numOfInitialKeywords: 0, + /** + * Get the service url based on options set in the dom. + * @return {String} A jsonp url for search suggestions with query missing from the end. + */ + getUrl :function(){ + // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js + // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins + // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list + // https://developers.google.com/custom-search/docs/ref_languages + services={ + "google": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=", + "google news": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=", + "google shopping": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=", + "google books": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=", + "youtube": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=", + "google videos": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=", + "google images": + "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=", + "yahoo": + "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=", + "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=", + "ebay": + "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=", + "amazon": + "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=", + "twitter": + "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=", + "baidu": "http://suggestion.baidu.com/su?cb=?&wd=", + "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=", + }; + + return _.template(services[this.options.service])(this.options); + }, + /** Parse response per service **/ + parseServiceResponse: function(res){ + // Each take a json response tand return a keyword array + RESPONSE_TEMPLATES = { + // opensearch default + "default": function (res) { + return res[1]; + }, + "yahoo": function (res) { + return _.map(res.gossip.results, 'key'); + }, + "ebay": function (res) { + return res.res ? res.res.sug : []; + }, + "twitter": function (res) { + return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick); + }, + "baidu": function (res) { + return res.s; + }, + "yandex": function(res){ + return _.map(res[1], function(r){ + return typeof r === 'string' ? r : r[1]; + }); + } + }; + var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"]; + return parser(res); + }, -var table; -var prefixes; -var suffixes; -var objectStore; + setUpDb: function(success){ + var self=this; + // setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB + var dbReq = window.indexedDB.open("KeywordShitter2", 3); + /** Error handler for all child transactions as events bubbe up **/ + dbReq.onerror = function (event) { + return console.error('Error opening indexedDB database KeywordShitter2', event); + }; + dbReq.onsuccess = function (event) { + self.db = event.target.result; + self.db.onerror = function (event) { + // Generic error handler requests + if (event && event.target && event.target.error) + console.error("Database error",event.target.error.errorCode, event.target.error.message, event); + else + console.error("Database error",arguments); + return this; + }; + if (success) success(self.db,dbReq); + return self.db; + }; + dbReq.onupgradeneeded = function (event) { + console.log("running onupgradeneeded"); + self.db = event.target.result; -var myIp; + if (!self.db.objectStoreNames.contains("suggestions")) { + var objectStore = self.db.createObjectStore("suggestions", { + autoIncrement: true, keyPath: 'id' + }); + // Create an index to search suggestions by + // he query that prompted the suggestion + if (!objectStore.indexNames.contains('search')) + objectStore.createIndex("search", "search", {unique: false}); + + // and by suggestion/keyword + if (!objectStore.indexNames.contains('keyword')) + objectStore.createIndex("keyword", "keyword", {unique: false}); + + } + + return self.db; + }; + return dbReq; + }, + + toggleWork: function(){ + if (this.doWork === false) + this.StartWork(); + else + this.StopWork(); + }, + + StartWork: function() { + if (this.doWork === false) { + // reset these + this.saveSettings(); + $('#startjob').val('Stop Job').text('Stop shitting').addClass('btn-danger'); + this.hashMapInputs = {}; + this.keywordsToQuery = []; + this.keywordsToQueryIndex = 0; + + this.hashMapInputs[""] = true; + this.hashMapInputs[" "] = true; + this.hashMapInputs[" "] = true; + + // update config + this.options = this.getOptions(); + + // get queries from the input + var ks = $('#input').val().split("\n"); + this.keywordsToQuery=ks.map(this.CleanVal); -$.getJSON('https://api.ipify.org?format=json', function (data) { - myIp = data.ip; -}); -// $.getJSON("http://jsonip.com?callback=?", function (data) { -// myIp = data.host; -// }); + // add variations of the initial terms + // (before we start adding variations of the results) + if (!this.keywordsToQuery.length) + this.permuteResultsToQueue([' ']); + else { + var untickedInputs = this.keywordsToQuery.filter(function(k){ + return k.slice(-1)!=='✓' && k.slice(-1)!=='❌'; + }); + this.permuteResultsToQueue(untickedInputs); + } + this.numOfInitialKeywords = this.keywordsToQuery.length; + // show the extended queue + this.FilterAndDisplay(); + this.doWork = true; + this.progress1.start(); + // $('#input').hide(); + // $('#advanced').collapse("hide"); -// TODO Implement alternative services -// Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js -// Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins -services={ - "google": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&q=${query}&ds=${site}&gl=${country}", - "yahoo": - "https://search.yahoo.com/sugg/ff?output=fxjson&appid=ffd&command=${query}", - "bing": "http://api.bing.com/osjson.aspx?query=${query}", - "ebay": - "http://autosug.ebay.com/autosug?kwd=${query}&_jgr=1&sId=0&_ch=0&callback=nil", - "amazon": - "http://completion.amazon.co.uk/search/complete?method=completion&q=${query}&search-alias=aps&mkt=4", - "twitter": - "https://twitter.com/i/search/typeahead.json?count=${max_results}&q=${query}&result_type=topics&src=SEARCH_BOX" - }; -function ebayParser(){} - // s = req.lstrip('/**/nil/(').rstrip(')') - // sugg_texts = json.loads(s)['res']['sug'] - // print('j', sugg_texts) - // return {'sugg_texts': sugg_texts} - - -function twitterParser(){} - // j = json.loads(req) - // return { - // 'sugg_texts': [t['topic'] for t in j['topics']], - // 'meta': j, - // 'relevances': [t['rounded_score'] for t in j['topics']], - // } -var RESPONSE_TEMPLATES = { - "google": ['query', 'sugg_texts', 'sugg_titles', '_', 'meta'], - "google_maps": ['query', 'sugg_texts', 'sugg_titles', '_', 'meta'], - "yahoo": ['query', 'sugg_texts'], - "bing": ['query', 'sugg_texts'], - "bing_search": ['query', 'sugg_texts'], - "ebay": ebayParser, - "amazon": ['query', 'sugg_texts'], - "twitter": twitterParser -}; - -// setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB - -/** Basic error handler **/ -function errorHandler(){ - console.error(this,arguments); - return this; // for chaining -} -var db; -var dbReq = window.indexedDB.open("KeywordShitter2", 2); -dbReq.onerror = function (event) { - console.error('dbReq', event); -}; -dbReq.onsuccess = function (event) { - // Do something with request.result! - console.log('dbReq', event); - db = event.target.result; - db.onerror = function (event) { - // Generic error handler for all errors targeted at this database's - // requests! - console.error("Database error: " + event.target.errorCode); - }; -}; -dbReq.onupgradeneeded = function (event) { - console.log("running onupgradeneeded"); - db = event.target.result; - - if (!db.objectStoreNames.contains("suggestions")) { - objectStore = db.createObjectStore("suggestions", { - autoIncrement: true - }); - - // Create an index to search suggestions by - // he query that prompted the suggestion - objectStore.createIndex("search", "search", { - unique: false - }); - // and by suggestion - objectStore.createIndex("keyword", "keyword", { - unique: false - }); - - } else { - // objectStore = db.objectStore("customers"); - } -}; - -window.setInterval(DoJob, 750); - -function StartJob() { - if (doWork === false) { - hashMapResults = {}; - keywordsToQuery = []; - keywordsToQueryIndex = 0; - - hashMapResults[""] = 1; - // hashMapResults[" "] = 1; - // hashMapResults[" "] = 1; - - // update config - prefixes = $('#prefixes').val().split(','); - suffixes = $('#suffixes').val().split(','); - - var ks = $('#input').val().split("\n"); - var i = 0; - for (i = 0; i < ks.length; i++) { - keywordsToQuery[keywordsToQuery.length] = ks[i]; - - } - numOfInitialKeywords = keywordsToQuery.length; - FilterAndDisplay(); - - doWork = true; - $('#startjob').val('Stop Job').text('Stop shitting').addClass('btn-danger'); - // $('#input').hide(); - - } else { - doWork = false; - $('#startjob').val('Start Job').text('Start shitting').removeClass('btn-danger'); - // $('#input').show(); - FilterAndDisplay(); - table.draw(); - table.columns.adjust(); - saveSettings(); - } -} - -function DoJob() { - if (doWork === true && queryLock === false) { - if (keywordsToQueryIndex < keywordsToQuery.length) { - var currentKw = keywordsToQuery[keywordsToQueryIndex]; - if (currentKw[currentKw.length - 1] != '✓') { - QueryKeyword(currentKw); - keywordsToQueryIndex++; } else { - // we didn't do a query immediatly go to next query - keywordsToQueryIndex++; - DoJob(); - } - } else { - if (numOfInitialKeywords != keywordsToQuery.length) { - doWork = false; + } + }, + + StopWork: function(){ + if (this.doWork){ $('#startjob').val('Start Job').text('Start shitting').removeClass('btn-danger'); - $('#input').show(); - FilterAndDisplay(); - table.draw(); - table.columns.adjust(); - } else { - keywordsToQueryIndex = 0; + this.doWork = false; + // $('#input').show(); + this.table.draw(); + this.table.columns.adjust(); + this.saveSettings(); + this.FilterAndDisplay(); + this.progress1.end(); } - } - } -} + }, -/** Make permutations of results and add to queue **/ -function permuteResultsToQueue(retList, search){ + DoJob: function() { + if (this.doWork === true && this.queryLock === false) { + if (this.keywordsToQueryIndex < this.numOfInitialKeywords) { + var currentKw = this.keywordsToQuery[this.keywordsToQueryIndex]; + if (currentKw.slice(-1)!=='✓' && currentKw.slice(-1)!=='❌') { + this.QueryKeyword(currentKw); + this.keywordsToQueryIndex++; + } else { + // we didn't do a query immediatly go to next query + this.keywordsToQueryIndex++; + this.DoJob(); + } - // sort so the shortest is first in the queue - retList.sort(function (a, b) { - return a.length - b.length; - }); + var prog = parseInt(this.keywordsToQueryIndex/this.numOfInitialKeywords*100); + this.progress1.set(prog); + this.FilterAndDisplay(); - for (var i = 0; i < retList.length; i++) { - var cleanKw = CleanVal(retList[i]); - if (cleanKw.length && !hashMapResults[cleanKw]){ - hashMapResults[cleanKw] = 1; - - // add base suggestion to queue - if (cleanKw!==search) - keywordsToQuery[keywordsToQuery.length] = cleanKw; - - // add prefix permutations - for (var k = 0; k < prefixes.length; k++) { - var chr = prefixes[k]; - var currentx = chr + ' ' + cleanKw; - keywordsToQuery[keywordsToQuery.length] = currentx; - hashMapResults[currentx] = 1; - } - // add suffix permutations - for (var j = 0; j < prefixes.length; j++) { - var chr = prefixes[j]; - var currentx = cleanKw + ' ' + chr; - keywordsToQuery[keywordsToQuery.length] = currentx; - hashMapResults[currentx] = 1; - } - } - } -} - -/** Display data from db upon pressing load button **/ -function loadFromDB(){ - var reqObj = db.transaction(["suggestions"],"readonly"). - objectStore("suggestions") - .getAll() - .onsuccess = function(e) { - if (e.target.result.length){ - table.data(e.target.result); - var data =[]; - for (var i = 0; i < e.target.result.length; i++) { - var d = e.target.result[i]; - data.push([i,d.keyword,d.Length,null,null,d.search]); + } else { + if (this.options.keepRunning) { + console.log('finish initial queue'); + this.StopWork(); + this.StartWork(); + } else { + console.log('finish initial queue'); + this.StopWork(); + } } - table.rows.add(data); } - table.draw(false); - }; - reqObj.onerror=errorHandler; -} + }, -/** Display results **/ -function displayResults(retList, search, dontDisplay){ + addResultsToQueue: function(retList, search){ + retList=retList.map(this.CleanVal); + // add each result to list first before permutations + for (var j = 0; j < retList.length; j++) { + cleanKw = retList[j]; + // add base suggestion to queue if it's not already done and isn't empty + if (cleanKw && cleanKw.length && !this.hashMapInputs[cleanKw] && this.keywordsToQuery.indexOf(cleanKw)===-1) + this.keywordsToQuery.push(cleanKw); + this.hashMapInputs[cleanKw] = true; + } + }, + /** Make permutations of results and add to queue **/ + permuteResultsToQueue: function(retList, search){ + var chr, currentx, currentKw; + var self = this; + this.hashMapInputs[search] = true; + // sort so the shortest is first in the queue TODO add option? + // retList.sort(function (a, b) { + // return a.length - b.length; + // }); - for (var i = 0; i < retList.length; i++) { - var cleanKw = CleanVal(retList[i]); + function addPrefix(s,prefix){ + return prefix+' '+s; + } + function addSuffix(s,suffix){ + return s+' '+suffix; + } + // clean + retList=retList.map(this.CleanVal); - // we get an annoying popup alert if we add undefined values, catch them - if (cleanKw===undefined||table.rows()[0].length===undefined||cleanKw.length===undefined,search===undefined){ - console.error('Undefined values',{ - id: table.rows()[0].length, - keyword: cleanKw, - length: cleanKw? cleanKw.length: undefined, - search:search - }); - continue; + // get permutations + var newInputs = retList.reduce(function(result, keyword){ + return _.concat( + result, + self.options.prefixes.map(addPrefix.bind(self,keyword)), + self.options.suffixes.map(addSuffix.bind(self,keyword)) + ); + }, []); - } else if (cleanKw!==undefined && cleanKw.length){ - // Check if suggestion is already displayed before adding - // var matches = table.data().filter(function(v){return v[1]===cleanKw && v[5]==search;}).count(); - // if (!matches) - table.row.add([ - table.rows()[0].length, - cleanKw, - cleanKw.length, - null, - null, - search]); - } - } - if (!dontDisplay) table.draw(false); -} + // add to queue + this.keywordsToQuery=_.concat(this.keywordsToQuery,newInputs); -/** Store new results in db and hashmap **/ -function storeResults(retList, search, url){ + return newInputs; + }, - for (var i = 0; i < retList.length; i++) { - var cleanKw = CleanVal(retList[i]); - if (cleanKw.length){ - // TODO check if I should add in bulk? + /** export db as json **/ + exportDB: function(success){ + var reqObj = this.db.transaction(["suggestions"],"readonly") + .objectStore("suggestions") + .getAll(); + reqObj.onsuccess = function(e) { + var blob, name; + if (e.target.result.length){ + var jsonData = JSON.stringify(e.target.result); + blob = new Blob([jsonData], {type: "octet/stream"}); + var timeStamp = (new Date()).toISOString().replace(/[:\.]/g,'_').slice(0,-5); + name = 'keywordshitter_'+timeStamp+'_r'+e.target.result.length+'.json'; + saveAs(blob,name); + if (success instanceof Function) success(blob); + } + return blob; + }; + return reqObj; + }, - // add to db - var transaction = db.transaction(["suggestions"], "readwrite"); - transaction.onerror = errorHandler; - var objectStore = transaction.objectStore("suggestions"); - addReq = objectStore.add({ - keyword: cleanKw, - Length: cleanKw.length, - search: search, - ip: myIp, - url: this.url, - time: (new Date()).toUTCString() - }); - addReq.onerror=errorHandler; - } + clearDB: function(){ + this.db.transaction(["suggestions"], "readwrite") + .objectStore("suggestions") + .clear(); + console.warn('cleared all indexedDB data'); + }, - } -} + /** Display data from db upon pressing load button **/ + loadFromDB: function(){ + var self = this; + var reqObj = this.db.transaction(["suggestions"],"readonly") + .objectStore("suggestions") + .getAll() + .onsuccess = function(e) { + if (e.target.result.length){ + /// grab the fields we want + var data =[]; + for (var i = 0; i < e.target.result.length; i++) { + var da = e.target.result[i]; -/** mark a search as done in the queue **/ -function markAsDone(search){ - // mark as done in queue - var found=false; - for (var l = 0; l < keywordsToQuery.length; l++) { - if (keywordsToQuery[l]==search){ - keywordsToQuery[l]+=' ✓'; - found=true; - break; - } - } - if (!found){console.error('Did not find ', search, 'in queue');} -} + // also remove undefined so datatables doesn't bring up alerts + da = _.mapValues(da, function(v){return v===undefined ? null: v;}); -/** Get search suggestions for a keyword **/ -function QueryKeyword(search) { - var querykeyword = search; - var queryresult = ''; - queryLock = true; + // parse nums + // da = da.map(v => /^[\d+\.]+$/.test(v) ? Number(v): v); + // + data.push(da); + } + self.table.rows.add(data); + return self.table.draw(false); + } + }; + return; + }, - // first check in db - var reqObj = db.transaction(["suggestions"],"readonly"). - objectStore("suggestions") - .index("search") - .getAll(search) - .onsuccess = function(e) { - // console.log(e.target.result); - if (e.target.result.length){ - // search was done previously so display results from db - var retList = []; - for (var i = 0; i < e.target.result.length; i++) { - retList.push(e.target.result[i].keyword); - } - displayResults(retList,search); - markAsDone(search); - permuteResultsToQueue(retList); - queryLock = false; + /** Display results **/ + displayResults: function(retList, search, dontDisplay, url,data){ - // we didn't do a query immediatly go to next query - DoJob(); + var rows=[]; + retList=retList.map(this.CleanVal); + for (var i = 0; i < retList.length; i++) { + var cleanKw = retList[i]; + + // url might be in retlist + if (url===undefined) url=data[i].url; + + var da = { + id: this.table.rows()[0].length+i, + keyword: cleanKw, + length: cleanKw.length, + words: cleanKw.trim().split(/ +/).length, + volume: null, + cpc: null, + search: search, + domain: this.extractDomain(url) + }; + + // remove undefined values to avoid datatable alerts + da = _.mapValues(da, function(v){return v===undefined ? null: v;}); + + // TODO Check if suggestion is already displayed before adding + // var matches = table.data().filter(function(v){return v[1]===cleanKw && v[5]==search;}).count(); + // if (!matches) + rows.push(da); + } + this.table.rows.add(rows); + // if table is large lets defer rending to end to speed it up + if (!dontDisplay && this.table.data().length -1) { + domain = url.split('/')[2]; } else { - // search not done, lets do the query - $.ajax({ - url: "http://suggestqueries.google.com/complete/search", - jsonp: "jsonp", - dataType: "jsonp", - data: { - q: search, - client: "chrome" - }, - success: function (res, statusText, jqXHR) { - // var search = res[0]; - var retList = res[1]; - var char, currentx; - - storeResults(retList, search, this.url); - displayResults(retList, search); - permuteResultsToQueue(retList); - markAsDone(search); - - queryLock = false; - - }, - error: function(){ - queryLock = false; - } - }); + domain = url.split('/')[0]; } - }; - reqObj.onerror=errorHandler; -} -/** Clean input, may not all be needed **/ -function CleanVal(input) { - var val = input; + //find & remove port number + domain = domain.split(':')[0]; - // legacy - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("", ""); - // val = val.replace("&", "&"); - // val = val.replace("", ""); - // val = val.replace("&", ""); - // val = val.replace("'", "'"); - // val = val.replace("#39;", "'"); - // val = val.replace("", ""); - // val = val.replace("–", "2013"); + // custom, add ds= param to distinguish googles etc diff searchs + var mr = url.match('ds=(..?)&'); + if (mr && mr[1] && mr[1].length) domain+='&ds='+mr[1]; - // this removes navigation suggestions - if (val.length > 4 && val.substring(0, 4) == "http") val = ""; - return val; -} + var mr = url.match('gl=(..?)&'); + if (mr && mr[1] && mr[1].length) domain+='&gl='+mr[1]; -/** TODO get this working **/ -function Filter(listToFilter) { - var retList = listToFilter; + var mr = url.match('hl=(..?)&'); + if (mr && mr[1] && mr[1].length) domain+='&hl='+mr[1]; - if ($("#filter-positive").val().length > 0) { - var filteredList = []; - var filterContains = $("#filter-positive").val().split("\n"); - for (var i = 0; i < retList.length; i++) { - var currentKeyword = retList[i]; - var boolContainsKeyword = false; - for (var j = 0; j < filterContains.length; j++) { - if (filterContains[j].length > 0) { - if (currentKeyword.indexOf(filterContains[j]) != -1) { - boolContainsKeyword = true; - break; + // lang code for yandex + var mr = url.match('uil=(..?)&'); + if (mr && mr[1] && mr[1].length) domain+='&uil='+mr[1]; + + return domain; + }, + + /** Store new results in db and hashmap **/ + storeResults: function(retList, search, url){ + var self = this; + retList=retList.map(this.CleanVal); + + // We will add the items async in order + // Ref: http://stackoverflow.com/a/13666741/221742 + var transaction = this.db.transaction(["suggestions"], "readwrite"); + var store = transaction.objectStore("suggestions"); + var i=0; + addNext(); + + /** Like an async for loop to add each to db **/ + function addNext() { + if (i 0) { - var filteredList = []; - var filterContains = $("#filter-negative").val().split("\n"); - for (var i = 0; i < retList.length; i++) { - var currentKeyword = retList[i]; - var boolCleanKeyword = true; - for (var j = 0; j < filterContains.length; j++) { - if (filterContains[j].length > 0) { - if (currentKeyword.indexOf(filterContains[j]) >= 0) { - boolCleanKeyword = false; - break; + /** mark a search as done in the queue **/ + markAsNone: function(search){ + // mark as done in queue + if (this.keywordsToQuery[this.keywordsToQueryIndex]===search) + this.keywordsToQuery[this.keywordsToQueryIndex]+=' ❌'; + else if (this.keywordsToQuery[this.keywordsToQueryIndex-1]===search) + this.keywordsToQuery[this.keywordsToQueryIndex-1]+=' ❌'; + else + console.warn('Cant find ',search,'in keywordsToQuery'); + }, + + /** Get search suggestions for a keyword **/ + QueryKeyword: function(search) { + var self = this; + this.queryLock = true; + + // first check in db + var reqObj = this.db.transaction(["suggestions"],"readonly") + .objectStore("suggestions") + .index("search") + .getAll(search); + reqObj.onsuccess = function(e) { + var domain = self.extractDomain(self.getUrl()); + var results = e.target.result.filter(function(r){return r.domain==domain;}); + // console.log(e.target.result); + if (results.length){ + // search was done previously so display results from db + var retList=_.map(results,'keyword'); + self.markAsDone(search); + self.displayResults(retList,search,undefined,undefined,results); + self.addResultsToQueue(retList); + if (self.options.keepRunning) self.permuteResultsToQueue(retList); + self.queryLock = false; + } + else { + // search not done, lets do the query + url = self.getUrl()+search; + $.ajax({ + url: url, + jsonp: "jsonp", + dataType: "jsonp", + success: function (res, statusText, jqXHR) { + + var retList = self.parseServiceResponse(res); + var char, currentx; + if (retList && retList.length){ + self.storeResults(retList, search, this.url); + self.displayResults(retList, search, undefined, this.url); + self.addResultsToQueue(retList); + if (self.options.keepRunning) self.permuteResultsToQueue(retList); + self.markAsDone(search); + } else { + // console.debug('No suggestions for query: "',search,'"'); + self.markAsNone(search); + } + self.queryLock = false; + return; + + }, + error: function(e){ + console.error(e); + self.queryLock = false; + return; + } + }); + } + }; + return reqObj; + }, + + /** Clean input, may not all be needed **/ + CleanVal: function(input) { + + function escapeHtml(unsafe) { + return unsafe + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + // google returns lowercase + input = input.toLowerCase(); + input = escapeHtml(input); + + // not sure if we want to trim, the search engines are sensitive to + // whitespace. (and also last search) + // input = input.trim(); + + // this removes navigation suggestions + if (input.length > 4 && input.substring(0, 4) == "http") input = ""; + return input; + }, + + /** TODO get this working **/ + Filter: function(listToFilter) { + var retList = listToFilter; + + if ($("#filter-positive").val().length > 0) { + var filteredList = []; + var filterContains = $("#filter-positive").val().split("\n"); + for (var i = 0; i < retList.length; i++) { + var currentKeyword = retList[i]; + var boolContainsKeyword = false; + for (var j = 0; j < filterContains.length; j++) { + if (filterContains[j].length > 0) { + if (currentKeyword.indexOf(filterContains[j]) != -1) { + boolContainsKeyword = true; + break; + } + } + } + + if (boolContainsKeyword) { + filteredList[filteredList.length] = currentKeyword; } } + + retList = filteredList; } - if (boolCleanKeyword) { - filteredList[filteredList.length] = currentKeyword; + if ($("#filter-negative").val().length > 0) { + var filteredList = []; + var filterContains = $("#filter-negative").val().split("\n"); + for (var l = 0; l < retList.length; l++) { + var currentKeyword = retList[l]; + var boolCleanKeyword = true; + for (var k = 0; k < filterContains.length; k++) { + if (filterContains[k].length > 0) { + if (currentKeyword.indexOf(filterContains[k]) >= 0) { + boolCleanKeyword = false; + break; + } + } + } + + if (boolCleanKeyword) { + filteredList[filteredList.length] = currentKeyword; + } + } + + retList = filteredList; } + + return retList; + }, + + /** display the queue, and update description of it **/ + FilterAndDisplay: function() { + var i = 0; + var sb = ''; + + var outputKeywords = this.keywordsToQuery; + for (i = 0; i < Math.min(outputKeywords.length,this.options.maxQueueDisplay); i++) { + sb += outputKeywords[i]; + sb += '\n'; + } + if (outputKeywords.length>this.options.maxQueueDisplay) sb+='...\n'; + $("#input").val(sb); + $("#numofkeywords").html('Queue:' + outputKeywords.length); + }, + + + /** overrides default with dom options with arguments options **/ + getOptions: function(argOptions){ + var defaultOptions={ + deferTableUpdatesAtRows: 5000, + keepRunning: false, + maxQueueDisplay: 5000, + country: "", + filterNegative: "", + filterPositive: "", + lang: "", + prefixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "how", "which", "why", "where", "who", "when", "are", "what"], + rateLimit: 750, + service: "google", + suffixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "like", "for", "without", "with", "versus", "vs", "to", "near", "except", "has"] + }; // for now defaults are set in html + if (argOptions===undefined) argOptions={}; + return _.defaults(argOptions,this.getDomOptions(),defaultOptions); + }, + + /** read settings from webpage **/ + getDomOptions: function(){ + + var service= $('#service').val(), + filterNegative = $('#filter-negative').val(), + filterPositive = $('#filter-positive').val(), + rateLimit = parseInt($('#rate-limit').val()), + // input: $('#input').val(), + prefixes = $('#prefixes').val(), + suffixes = $('#suffixes').val(), + country = $('#country').val(), + lang = $('#lang').val(), + keepRunning = $('#keep-running').prop('checked'); + if (prefixes && prefixes.length) + prefixes=prefixes.split(','); + else + prefixes=undefined; + if (suffixes && suffixes.length) + suffixes=suffixes.split(','); + else + suffixes=undefined; + + var options={}; + if (service) options.service=service; + if (filterNegative) options.filterNegative=filterNegative; + if (filterPositive) ooptions.filterPositive=filterPositive; + if (rateLimit) options.rateLimit=rateLimit; + if (prefixes) options.prefixes=prefixes; + if (suffixes) options.suffixes=suffixes; + if (country) options.country=country; + if (lang) options.lang=lang; + if (keepRunning) options.keepRunning=keepRunning; + return options; + }, + + /** load settings from localStorage **/ + loadSettings: function(){ + // TODO do table settings as well, e.g. column visibilitity + if (localStorage.service) $("#service").val( localStorage.service ); + if (localStorage.country) $('#country').val(localStorage.country); + if (localStorage.lang) $('#lang').val(localStorage.lang); + if (localStorage.filterNegative) $("#filter-negative").val( localStorage.filterNegative ); + if (localStorage.filterPositive) $("#filter-positive").val( localStorage.filterPositive ); + if (localStorage.rateLimit) $("#rate-limit").val( localStorage.rateLimit ); + if (localStorage.input) $("#input").val( localStorage.input ); + if (localStorage.prefixes) $("#prefixes").val( localStorage.prefixes ); + if (localStorage.suffixes) $("#suffixes").val( localStorage.suffixes ); + if (localStorage.keepRunning) $('#keep-running').prop('checked',localStorage.keepRunning=="true"); + + }, + /** save settings to localStorage. **/ + saveSettings: function(){ + localStorage.service = $('#service').val(); + localStorage.country = $('#country').val(); + localStorage.lang = $('#lang').val(); + localStorage.filterNegative = $('#filter-negative').val(); + localStorage.filterPositive = $('#filter-positive').val(); + localStorage.rateLimit = $('#rate-limit').val(); + localStorage.input = $('#input').val(); + localStorage.prefixes = $('#prefixes').val(); + localStorage.suffixes = $('#suffixes').val(); + localStorage.keepRunning = $('#keep-running').prop('checked'); + }, + + /** reset inputs and results, but not settings **/ + reset: function(){ + this.table.clear(); + this.table.draw(); + $('#input').val(''); + this.saveSettings(); + }, + + init: function(){ + this.setUpDb(); + this.loadSettings(); + this.options = this.getOptions(); + + window.setInterval(this.DoJob.bind(this), this.options.rateLimit); + + $('#progress1').addClass('progressjs-progress'); + this.progress1 = progressJs("#progress1"); + + + $('#startjob').on('click',this.toggleWork.bind(this)); + $('#reset').on('click',this.reset.bind(this)); + $('#load-from-cache').on('click',this.loadFromDB.bind(this)); + $('#export-from-cache').on('click',this.exportDB.bind(this)); + $('#clear-cache').on('click',this.clearDB.bind(this)); + + + this.table = $('#outtable').DataTable({ + pageLength: 25, + "lengthMenu": [ 10, 25, 50, 75, 100,800], + dom: + "<'row'<'col-sm-5'B><'col-sm-7'<'pull-right'p>>>" + + "<'row'<'col-sm-8'i><'col-sm-4'<'pull-right'f>>>" + + "<'row'<'col-sm-12'tr>>", + buttons: [ + 'colvis', + 'pageLength', + { + extend: 'collection', + text: 'Export', + buttons: [ + 'copyHtml5', + 'csvHtml5', + { + extend: 'csvHtml5', + fieldBoundary: "", + text: 'Copy keywords', + // 'customize': function(data,options){return data.split('\n').join(',');}, + header: false, + exportOptions: { + stripNewlines: true, + stripHtml: true, + decodeEntities: true, + columns: 'keyword', + // format:{ + // body: function(html,i){console.log(html);return html} + // } + } + }, + ] + }, + + ], + "columnDefs": [ + { + "title": "id", + "data": "id", + "targets": 0, + "visible": false, + }, { + "title": "Keyword", + "data": "keyword", + "responsivePriority": 1, + "targets": 1, + }, { + "title": "Length", + "data": "length", + "targets": 2, + "visible": false, + "type": "num" + }, { + "name": "volume", + "data": "volume", + "targets": 3, + "visible": false, + "type": "num" + }, { + "title": "CPC", + "data": "cpc", + "targets": 4, + "visible": false, + "type": "num" + }, { + "title": "Search", + "data": "search", + "responsivePriority": 3, + "targets": 5, + "visible": false, + }, { + "title": "Domain", + "data": "domain", + "responsivePriority": 2, + "targets": 6, + "visible": false, + }, { + "title": "Words", + "data": "words", + "targets": 7, + "visible": false, + "type": "num" + }, + ], + order: [[ 0, 'desc' ]], + // colReorder: {}, + stateSave: true, + "bDeferRender": true, + // fixedHeader: true, + // responsive: { + // details: { + // type: 'column', + // target: 'tr' + // }, + // }, + // scrollY: 500, + // deferRender: true, + // scroller: true + }); + + $.getJSON('https://api.ipify.org?format=json', function (data) { + this.myIp = data.ip; + }); + } - - retList = filteredList; - } - - return retList; -} - -function FilterAndDisplay() { - var i = 0; - var sb = ''; - var outputKeywords = Filter(keywordsToQuery); - for (i = 0; i < outputKeywords.length; i++) { - sb += outputKeywords[i]; - sb += '\n'; - } - $("#input").val(sb); - $("#numofkeywords").html('Queue:' + outputKeywords.length + ', Results: ' + table.data().length); -} - -/** read settings from webpage **/ -// function readSettings(){ -// rateLimit = $('#service').val(); -// filterNegative = $('#filter-negative').val(); -// filterPositive = $('#filter-positive').val(); -// rateLimit = $('#rate-limit').val(); -// // input = $('#input').val(); -// prefixes = $('#prefixes').val(); -// suffixes = $('#suffixes').val(); -// } -/** load settings from localStorage **/ -function loadSettings(){ - // TODO do table settings as well, e.g. column visibilitity - if (localStorage.service) $("#service").val( localStorage.service ); - if (localStorage.filterNegative) $("#filter-negative").val( localStorage.filterNegative ); - if (localStorage.filterPositive) $("#filter-positive").val( localStorage.filterPositive ); - if (localStorage.rateLimit) $("#rate-limit").val( localStorage.rateLimit ); - if (localStorage.input) $("#input").val( localStorage.input ); - if (localStorage.prefixes) $("#prefixes").val( localStorage.prefixes ); - if (localStorage.suffixes) $("#suffixes").val( localStorage.suffixes ); - -} -/** save settings to localStorage. **/ -function saveSettings(){ - localStorage.service = $('#service').val(); - localStorage.filterNegative = $('#filter-negative').val(); - localStorage.filterPositive = $('#filter-positive').val(); - localStorage.rateLimit = $('#rate-limit').val(); - localStorage.input = $('#input').val(); - localStorage.prefixes = $('#prefixes').val(); - localStorage.suffixes = $('#suffixes').val(); -} -function reset(){ - table.clear(); - table.draw(); - $('#input').val(''); - saveSettings(); -} - -$(document).ready(function () { - loadSettings(); - table = $('#outtable').DataTable({ - pageLength: 25, - dom: - "<'row'<'col-sm-5'B><'col-sm-7'<'pull-right'p>>>" + - "<'row'<'col-sm-8'i><'col-sm-4'<'pull-right'f>>>" + - "<'row'<'col-sm-12'tr>>", - buttons: ['copyHtml5', 'csvHtml5','colvis','pageLength'], - "columnDefs": [ - { - "name": "id", - "targets": 0, - "visible": false, - }, { - "name": "keyword", - "targets": 1 - }, { - "name": "length", - "targets": 2, - "visible": false, - }, { - "name": "volume", - "targets": 3, - "visible": false, - }, { - "name": "cpc", - "targets": 4, - "visible": false, - }, { - "name": "search", - "targets": 5, - "visible": false, - }], - order: [[ 0, 'desc' ]], - colReorder: {}, - stateSave: true - }); -}); + }; +}(); diff --git a/package.json b/package.json index e14a73c..1ea7d47 100644 --- a/package.json +++ b/package.json @@ -2,12 +2,21 @@ "name": "keywordshitter2", "version": "0.0.5", "description": "The second generation of keyword shitter. Bigger, better, and more pungent than other products.", - "main": "public/index.html", + "main": "server.js", + "repository": "https://github.com/wassname/keywordshitter2", "scripts": { - "test": "mocha", + "test": "public/test.html", "build-css": "lessc ./src/less/main.less > public/css/style.css", - "build": "npm run build-css" + "build": "npm run build-css", + "start": "node server.js" }, - "author": "", - "license": "MIT" + "author": "blackmia,wassname", + "license": "MIT", + "devDependencies": { + "chai": "^3.5.0", + "cors": "^2.7.1", + "express": "^4.13.4", + "lessc": "latest", + "mocha": "^2.4.5" + } } diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000..b421021 Binary files /dev/null and b/public/favicon.ico differ diff --git a/public/fonts/glyphicons-halflings-regular.eot b/public/fonts/glyphicons-halflings-regular.eot new file mode 100644 index 0000000..b93a495 Binary files /dev/null and b/public/fonts/glyphicons-halflings-regular.eot differ diff --git a/public/fonts/glyphicons-halflings-regular.svg b/public/fonts/glyphicons-halflings-regular.svg new file mode 100644 index 0000000..94fb549 --- /dev/null +++ b/public/fonts/glyphicons-halflings-regular.svg @@ -0,0 +1,288 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/public/fonts/glyphicons-halflings-regular.ttf b/public/fonts/glyphicons-halflings-regular.ttf new file mode 100644 index 0000000..1413fc6 Binary files /dev/null and b/public/fonts/glyphicons-halflings-regular.ttf differ diff --git a/public/fonts/glyphicons-halflings-regular.woff b/public/fonts/glyphicons-halflings-regular.woff new file mode 100644 index 0000000..9e61285 Binary files /dev/null and b/public/fonts/glyphicons-halflings-regular.woff differ diff --git a/public/fonts/glyphicons-halflings-regular.woff2 b/public/fonts/glyphicons-halflings-regular.woff2 new file mode 100644 index 0000000..64539b5 Binary files /dev/null and b/public/fonts/glyphicons-halflings-regular.woff2 differ diff --git a/public/js/test.js b/public/js/test.js new file mode 100644 index 0000000..e27081a --- /dev/null +++ b/public/js/test.js @@ -0,0 +1,156 @@ +var expect = chai.expect; +var assert = chai.assert; + +// TODO +describe('permute', function(){ + before(function() { + KWS.hashMapInputs = {}; + KWS.keywordsToQuery = []; + }); + + after(function() { + KWS.hashMapInputs = {}; + KWS.keywordsToQuery = []; + }); + + it('should correctly permute results and add to queue', function() { + var retList = ['a','b','c d',' a longer one']; + KWS.permuteResultsToQueue(retList,'test'); + assert(KWS.keywordsToQuery.length>0); + assert(KWS.hashMapInputs['b']); + assert(KWS.hashMapInputs['test']); + // var options = KWS.getOptions(); + // var expectedLength = options.prefixes.length*retList.length+options.suffixes.length*retList.length + // assert(KWS.keywordsToQuery.length==expectedLength,''+expectedLength+'!='+KWS.keywordsToQuery.length); + }); +}); + +// TODO use localForage or most popular promise wrapper for indexedDB then test +// describe('exportDB', function(){ +// }); + +describe('db', function() { + it('should correctly set up db', function(done) { + var dbReq=KWS.setUpDb(); + var oldonsuccess=dbReq.onsuccess; + dbReq.onsuccess=function(event){ + // override onld onsucess because it doesn't use promises + var db=oldonsuccess.bind(dbReq)(event); + assert(db); + assert(db.objectStoreNames.contains('suggestions'),'db should contain suggestons store'); + var indexNames =db.transaction('suggestions','readonly').objectStore('suggestions').indexNames; + assert(indexNames.contains('keyword','search')); + done(); + }; + }); +}); + + +describe('extractDomain', function(){ + + for (var expectedDomain in testData.domainUrls) { + if (testData.domainUrls.hasOwnProperty(expectedDomain)) { + it('should correctly extract domain from url, domain='+expectedDomain, function() { + var url = testData.domainUrls[expectedDomain]; + var domain = KWS.extractDomain(url); + assert.equal(expectedDomain,domain); + }); + } + } + +}); + + +describe('responses', function(){ + for (var service in testData.responses) { + if (testData.responses.hasOwnProperty(service)) { + it('should correctly parse test data for service='+service, function() { + var res = testData.responses[service]; + var options = {service:service}; + var parsedRes = KWS.parseServiceResponse(res,options); + + assert.typeOf(parsedRes,'array'); + expect(parsedRes).to.have.length.above(1); + parsedRes.map(r => assert.typeOf(r, 'string')); + }); + } + } +}); + +describe('services', function() { + 'use strict'; + // TODO test for each language and country and assert results!=default res + this.timeout(10000); + this.slow(2000); + + // data + var services = ["google", "google news", "google shopping", "google books", "youtube", "google videos", "google images", "yahoo", "bing", "ebay", "amazon", "twitter", "baidu", "yandex"]; + var searches = ["where"]; + var searchesDifficult = [" * ", "❥"]; + + + before(function() { + + }); + + after(function() { + + }); + + services.forEach(function(service){ + searches.forEach(function(search){ + it('should correctly get and parse data'+'. Service="'+service+'" search="'+search+'"', function(done) { + var options = {service:service}; + var url = KWS.getUrl(options)+search; + return $.ajax({ + url: url, + async: false, + jsonp: "jsonp", + dataType: "jsonp", + success: function (res, statusText, jqXHR) { + assert(statusText=="success"); + assert(res!==[]); + var parsedRes = KWS.parseServiceResponse(res,options); + assert.typeOf(parsedRes,'array'); + parsedRes.map(r => assert.typeOf(r, 'string')); + done(); + }, + error: function(jqXHR,textStatus,err){ + done(err); + return err; + } + }); + }); + }); + }); + + + services.forEach(function(service){ + searchesDifficult.forEach(function(search){ + it('should get and parse difficult data'+'. Service="'+service+'" search="'+search+'"', function(done) { + var options = {service:service}; + var url = KWS.getUrl(options)+search; + return $.ajax({ + url: url, + async: false, + jsonp: "jsonp", + dataType: "jsonp", + success: function (res, statusText, jqXHR) { + assert(statusText=="success"); + assert(res!==[]); + + var parsedRes = KWS.parseServiceResponse(res,options); + assert.typeOf(parsedRes,'array'); + parsedRes.map(r => assert.typeOf(r, 'string')); + return done(); + + }, + error: function(jqXHR,textStatus,err){ + return done(err); + } + }); + }); + }); + }); + +}); diff --git a/public/js/testData.js b/public/js/testData.js new file mode 100644 index 0000000..b95e701 --- /dev/null +++ b/public/js/testData.js @@ -0,0 +1,568 @@ +var testData = function () { + return { + domainUrls: { + "api.bing.com": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=jQuery111&query=a &_=11111", + "autosug.ebay.com": "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=jQuery111&kwd=a &_=11111", + "completion.amazon.co.uk": "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=jQuery111&q=a &_=11111", + "search.yahoo.com": "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=jQuery111&command=a &_=11111", + "suggestqueries.google.com&gl=US&hl=en": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&gl=US&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.com&ds=i&gl=US&hl=en": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&ds=i&gl=US&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.com&ds=sh&gl=US&hl=en": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&ds=sh&gl=US&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.com&ds=yt&gl=US&hl=en": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&ds=yt&gl=US&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.com&gl=us&hl=en": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&gl=us&callback=jQuery111&q=oil&_=11111", + "suggestqueries.google.com&gl=us&hl=fi": "http://suggestqueries.google.com/complete/search?client=chrome&hl=fi&gl=us&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.com&gl=us&hl=he": "http://suggestqueries.google.com/complete/search?client=chrome&hl=he&gl=us&callback=jQuery111&q=a &_=11111", + "suggestqueries.google.combo": "http://suggestqueries.google.com/complete/search?client=chrome&hl=en&ds=bo&gl=US&callback=jQuery111&q=a &_=11111", + "twitter.com": "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=jQuery111&q=I accidentally&_=11111" + }, + responses: { + "yahoo": { + "gossip": { + "qry": "where", + "gprid": "xfbzDJDaR9yN9o2B4J0JBA", + "results": [{ + "key": "where's my refund", + "mrk": 6 + }, { + "key": "where is super bowl 2016", + "mrk": 6 + }, { + "key": "where's my refund 2016", + "mrk": 6 + }, { + "key": "where the wild things are", + "mrk": 6 + }, { + "key": "wheresgeorge.com", + "mrk": 5 + }, { + "key": "where's waldo", + "mrk": 6 + }, { + "key": "where to get a passport", + "mrk": 6 + }, { + "key": "where the red fern grows", + "mrk": 6 + }, { + "key": "where can i watch movies online for free", + "mrk": 6 + }, { + "key": "where are they now", + "mrk": 6 + }] + } + }, + "google": ["where", ["where's my refund", "where am i", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where is xur", "where is oj simpson now", "where is the next primary", "where to invade next", "where the wild things are", "where are you now", "where is dubai", "where is potomac", "where's waldo", "where is my phone", "where the heart is", "where the red fern grows", "where is the zika virus", "where was ted cruz born", "where is oak island", "where are you christmas", "where ya at"], + ["", "", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [1250, 602, 601, 600, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "google news": ["where", ["where is mali", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where is isis", "where is xur", "where is john cena", "where is ronda rousey", "where is isis now", "where is san bernardino", "where is luke skywalker", "where am i", "where she went movie", "where is josh duggar now", "where is el nino", "where is reid on criminal minds", "where will isis attack next", "where is isis located", "where is frank ocean", "where the hoes at", "where is spencer on criminal minds", "where is syria"], + ["", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [602, 601, 600, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "google books": ["where", ["where the red fern grows", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where the wild things are", "where she went", "where was this book published", "where the sidewalk ends", "where rainbows end", "where am i", "where are you going where have you been", "wherever you go there you are", "where to buy", "where the red fern grows book", "where the mountain meets the moon", "where can i buy this book", "where the mind is without fear", "where is mali", "where good ideas come from", "where'd you go bernadette", "where's waldo", "whereas"], + ["", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [602, 601, 600, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "youtube": ["where", ["where are you now justin bieber", "where ya at future", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where are you christmas", "where is the love", "where are you christmas faith hill", "where do the good boys go to hideaway", "where is my mind", "where they from", "where the hood at", "where are you now justin bieber lyrics", "where ya at future lyrics", "where is the love black eyed peas lyrics", "where have you been rihanna", "where is my mind piano", "wheresmychallenge", "wherever you will go the calling", "where you at", "where they at doe", "where them girls at"], + ["", "", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [950, 602, 601, 600, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "bing": ["where", ["whereis", "where is", "whereis.com.au", "where is maps directions", "whereis nsw", "whereis.com", "where is santa", "whereis maps", "where is it", "where am i", "whereis.com.au directions", "where is.com.au"]], + "google videos": ["where", ["where are you now justin bieber", "where ya at future", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where are you christmas", "where is the love", "where are you christmas faith hill", "where do the good boys go to hideaway", "where is my mind", "where they from", "where the hood at", "where are you now justin bieber lyrics", "where ya at future lyrics", "where is the love black eyed peas lyrics", "where have you been rihanna", "where is my mind piano", "wheresmychallenge", "wherever you will go the calling", "where you at", "where they at doe", "where them girls at"], + ["", "", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [950, 602, 601, 600, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "google shopping": ["where", ["where to buy elf on the shelf", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where to buy cards against humanity", "where to buy essential oils", "where to buy bean boozled", "where to buy hoverboard", "where the wild things are", "where to buy citric acid", "where to buy mistletoe", "where can i buy a hoverboard", "where can you buy bean boozled", "where to buy ugly christmas sweater", "where can i buy a lokai bracelet", "where can i buy elf on the shelf", "where to buy beard oil", "where can i buy mercer's wine ice cream", "where's waldo", "where in the world is carmen sandiego", "where the red fern grows", "where to buy garcinia cambogia"], + ["", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [602, 601, 600, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "ebay": { + "prefix": "where", + "dict": "0", + "res": { + "sug": ["where the wild things are", "where women create", "where the sidewalk ends", "where monsters dwell", "where eagles dare", "where the red fern grows", "where's waldo", "wheres mickey vera bradley", "wheres waldo book", "where the wild things are book"], + "categories": [ + [1093, "Children & Young Adults"], + [2624, "TV, Movie & Character Toys"] + ], + "isTopQuery": false + } + }, + "google images": ["where", ["where's waldo", "http://www.irs.gov/individuals/article/0,,id=96596,00.html", "where the wild things are", "where is mali", "where's wally", "where to shoot a deer", "where are you", "where is your liver", "where is syria", "where the red fern grows", "where is your appendix", "where are you now", "where is isis", "where is bora bora", "where is dubai", "where is the liver", "where the sidewalk ends", "where is the liver located", "where in the world is carmen sandiego", "where's the beef"], + ["", "Where's My Refund - It's Quick, Easy, and Secure.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], + [], { + "google:clientdata": { + "bpc": false, + "tlw": false + }, + "google:suggestrelevance": [602, 601, 600, 566, 565, 564, 563, 562, 561, 560, 559, 558, 557, 556, 555, 554, 553, 552, 551, 550], + "google:suggesttype": ["QUERY", "NAVIGATION", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY", "QUERY"], + "google:verbatimrelevance": 851 + } + ], + "amazon": ["where", ["where she went", "where the wild things are", "where the trail ends", "this is where i leave you", "where is wally", "where to invade next", "where we are", "how to be parisian wherever you are", "where hope grows", "where is waldo"], + [{ + "nodes": [{ + "name": "Fremdsprachige Bücher", + "alias": "english-books" + }, { + "name": "Kindle-Shop", + "alias": "digital-text" + }] + }, {}, {}, {}, {}, {}, {}, {}, {}, {}], + [] + ], + "twitter": { + "num_results": 30, + "users": [], + "topics": [{ + "topic": "#WhereWereYou", + "rounded_score": 93013, + "tokens": [{ + "token": "#wherewereyou" + }], + "inline": false + }, { + "topic": "wherever you are", + "rounded_score": 75887, + "tokens": [{ + "token": "wherever" + }, { + "token": "you" + }, { + "token": "are" + }], + "inline": false + }, { + "topic": "where to invade next", + "rounded_score": 64255, + "tokens": [{ + "token": "where" + }, { + "token": "to" + }, { + "token": "invade" + }, { + "token": "next" + }], + "inline": false + }, { + "topic": "#WhereToInvadeNext", + "rounded_score": 64255, + "tokens": [{ + "token": "#wheretoinvadenext" + }], + "inline": false + }, { + "topic": "where are you now", + "rounded_score": 63587, + "tokens": [{ + "token": "where" + }, { + "token": "are" + }, { + "token": "you" + }, { + "token": "now" + }], + "inline": false + }, { + "topic": "#WhereDealsHappen", + "rounded_score": 63587, + "tokens": [{ + "token": "#wheredealshappen" + }], + "inline": false + }, { + "topic": "#WheresRey", + "rounded_score": 52917, + "tokens": [{ + "token": "#wheresrey" + }], + "inline": false + }, { + "topic": "where", + "rounded_score": 51445, + "tokens": [{ + "token": "where" + }], + "inline": false + }, { + "topic": "Wheres the coke baby", + "rounded_score": 43482, + "tokens": [{ + "token": "wheres" + }, { + "token": "the" + }, { + "token": "coke" + }, { + "token": "baby" + }], + "inline": false + }, { + "topic": "whereslloyd", + "rounded_score": 43109, + "tokens": [{ + "token": "whereslloyd" + }], + "inline": false + }, { + "topic": "where is my mind", + "rounded_score": 43074, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "my" + }, { + "token": "mind" + }], + "inline": false + }, { + "topic": "where do you see yourself", + "rounded_score": 40908, + "tokens": [{ + "token": "where" + }, { + "token": "do" + }, { + "token": "you" + }, { + "token": "see" + }, { + "token": "yourself" + }], + "inline": false + }, { + "topic": "WHERE FUTURES END", + "rounded_score": 40908, + "tokens": [{ + "token": "where" + }, { + "token": "futures" + }, { + "token": "end" + }], + "inline": false + }, { + "topic": "where you at", + "rounded_score": 40908, + "tokens": [{ + "token": "where" + }, { + "token": "you" + }, { + "token": "at" + }], + "inline": false + }, { + "topic": "WhereTheBlowAt", + "rounded_score": 40908, + "tokens": [{ + "token": "wheretheblowat" + }], + "inline": false + }, { + "topic": "where are we gonna meet up", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "are" + }, { + "token": "we" + }, { + "token": "gonna" + }, { + "token": "meet" + }, { + "token": "up" + }], + "inline": false + }, { + "topic": "where are you at summer", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "are" + }, { + "token": "you" + }, { + "token": "at" + }, { + "token": "summer" + }], + "inline": false + }, { + "topic": "wheresanna", + "rounded_score": 35857, + "tokens": [{ + "token": "wheresanna" + }], + "inline": false + }, { + "topic": "where you at donna", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "you" + }, { + "token": "at" + }, { + "token": "donna" + }], + "inline": false + }, { + "topic": "where is justin bieber", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "justin" + }, { + "token": "bieber" + }], + "inline": false + }, { + "topic": "where the air is sweet", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "the" + }, { + "token": "air" + }, { + "token": "is" + }, { + "token": "sweet" + }], + "inline": false + }, { + "topic": "where is luke", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "luke" + }], + "inline": false + }, { + "topic": "where is harry styles", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "harry" + }, { + "token": "styles" + }], + "inline": false + }, { + "topic": "where to buy beats", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "to" + }, { + "token": "buy" + }, { + "token": "beats" + }], + "inline": false + }, { + "topic": "where are they", + "rounded_score": 35857, + "tokens": [{ + "token": "where" + }, { + "token": "are" + }, { + "token": "they" + }], + "inline": false + }, { + "topic": "where jc", + "rounded_score": 32980, + "tokens": [{ + "token": "where" + }, { + "token": "jc" + }], + "inline": false + }, { + "topic": "where's dylan o'brien", + "rounded_score": 32980, + "tokens": [{ + "token": "where" + }, { + "token": "s" + }, { + "token": "dylan" + }, { + "token": "o" + }, { + "token": "brien" + }], + "inline": false + }, { + "topic": "where is drenthe", + "rounded_score": 32980, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "drenthe" + }], + "inline": false + }, { + "topic": "where is anita", + "rounded_score": 32980, + "tokens": [{ + "token": "where" + }, { + "token": "is" + }, { + "token": "anita" + }], + "inline": false + }, { + "topic": "wheres my quarter", + "rounded_score": 32980, + "tokens": [{ + "token": "wheres" + }, { + "token": "my" + }, { + "token": "quarter" + }], + "inline": false + }], + "oneclick": [], + "hashtags": [], + "completed_in": 0.005, + "query": "where" + }, + "baidu": { + "q": "where", + "p": false, + "s": ["whereas", "where are you now", "where is the love", "where are you", "whereby", "where did you go", "wherever you are", "where did you sleep last night", "wherever", "where there is a will there is a way"] + }, + "yandex": ["where", ["where's my refund", "where is chuck norris", "where the wild things are", "where", "where's my refund 2013", "where is my tax refund", "wheresmyrefund irs refund status", "where's my state refund", "where is my state refund", "where's my water", ["", "where is my mind", { + "sg_weight": 3.69087e-9, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where the wild roses grow", { + "sg_weight": 1.58832e-9, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "wherever you go", { + "sg_weight": 1.35346e-9, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where have you been", { + "sg_weight": 1.30423e-9, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where them girls at", { + "sg_weight": 1.10014e-9, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where are you", { + "sg_weight": 9.90519e-10, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where did you sleep last night", { + "sg_weight": 8.15709e-10, + "search_cgi": [ + ["noreask", "1"] + ] + }], + ["", "where is the love", { + "sg_weight": 7.31462e-10, + "search_cgi": [ + ["noreask", "1"] + ] + }] + ], { + "r": 211, + "log": "sgtype:BBBBBBBBBBArtArtArtArtArtArtArtArt", + "continue": "'s" + }] + } + }; +}(); diff --git a/public/test.html b/public/test.html new file mode 100644 index 0000000..dc45486 --- /dev/null +++ b/public/test.html @@ -0,0 +1,40 @@ + + + + + + Mocha Tests + + + + + + + + + + + + + + + + + + + + + +
+ + diff --git a/src/less/main.less b/src/less/main.less index 4066125..befb033 100644 --- a/src/less/main.less +++ b/src/less/main.less @@ -12,7 +12,7 @@ // clean minimalist tables .table.dataTable { - min-height: 150px; + min-height: 600px; } .table { // Cells @@ -57,3 +57,7 @@ .pagination { margin: 0px; } +.dataTables_wrapper .dataTables_paginate .paginate_button { + margin: 0px; + padding: 0px; +}
idKeyword