@@ -395,16 +395,22 @@ lying internet
-
-
+
-
+
+
+
+
+
@@ -475,7 +481,7 @@ lying internet
diff --git a/public/js/main.js b/public/js/main.js
index 4904605..6c997f7 100644
--- a/public/js/main.js
+++ b/public/js/main.js
@@ -1,731 +1,804 @@
-// quick way to check if we already have keyword
-var hashMapInputs = {};
+var KWS = function(){
-// keeping track of queue
-var numOfInitialKeywords = 0;
-var keywordsToQuery = [];
-var keywordsToQueryIndex = 0;
+ return {
+ db:undefined,
+ table: undefined,
+ myIp: undefined,
+ options: {},
+ // flags
+ queryLock: false,
+ doWork: false,
+ // keeping track of queue
+ hashMapInputs: {},
+ keywordsToQuery: [],
+ keywordsToQueryIndex: 0,
+ numOfInitialKeywords: 0,
+ /**
+ * Get the service url based on options set in the dom.
+ * @return {String} A jsonp url for search suggestions with query missing from the end.
+ */
+ getUrl :function(){
+ // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
+ // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
+ // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
+ // https://developers.google.com/custom-search/docs/ref_languages
+ services={
+ "google":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
+ "google news":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
+ "google shopping":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
+ "google books":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
+ "youtube":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
+ "google videos":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
+ "google images":
+ "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
+ "yahoo":
+ "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
+ "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
+ "ebay":
+ "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
+ "amazon":
+ "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
+ "twitter":
+ "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
+ "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
+ "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part="
+ };
-// flags
-var doWork = false;
-var queryLock = false;
-
-var maxQueueDisplay = 1000;
-var db;
-var table;
-var prefixes;
-var suffixes;
-var myIp;
-
-
-
-
-/**
- * Get the service url based on options set in the dom.
- * @return {String} A jsonp url for search suggestions with query missing from the end.
- */
-function getUrl(options){
- // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
- // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
- // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
- // https://developers.google.com/custom-search/docs/ref_languages
- services={
- "google":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
- "google news":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
- "google shopping":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
- "google books":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
- "youtube":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
- "google videos":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
- "google images":
- "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
- "yahoo":
- "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
- "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
- "ebay":
- "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
- "amazon":
- "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
- "twitter":
- "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
- "baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
- "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part="
- };
-
- options = getOptions(options);
- return _.template(services[options.service])(options);
-}
-
-
-/** Parse response per service **/
-function parseServiceResponse(res,options){
- // Each take a json response tand return a keyword array
- RESPONSE_TEMPLATES = {
- // opensearch default
- "default": function (res) {
- return res[1];
+ return _.template(services[this.options.service])(this.options);
},
- "yahoo": function (res) {
- return _.map(res.gossip.results, 'key');
- },
- "ebay": function (res) {
- return res.res ? res.res.sug : [];
- },
- "twitter": function (res) {
- return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
- },
- "baidu": function (res) {
- return res.s;
- },
- "yandex": function(res){
- return _.map(res[1], function(r){
- return typeof r === 'string' ? r : r[1];
- });
- },
- };
- options = getOptions(options);
- var parser = RESPONSE_TEMPLATES[options.service] || RESPONSE_TEMPLATES["default"];
- return parser(res);
-}
-function setUpDb(success){
- // setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB
- var dbReq = window.indexedDB.open("KeywordShitter2", 3);
- /** Error handler for all child transactions as events bubbe up **/
- dbReq.onerror = function (event) {
- return console.error('Error opening indexedDB database KeywordShitter2', event);
- };
- dbReq.onsuccess = function (event) {
- db = event.target.result;
- db.onerror = function (event) {
- // Generic error handler requests
- if (event && event.target && event.target.error)
- console.error("Database error",event.target.error.errorCode, event.target.error.message, event);
+
+ /** Parse response per service **/
+ parseServiceResponse: function(res){
+ // Each take a json response tand return a keyword array
+ RESPONSE_TEMPLATES = {
+ // opensearch default
+ "default": function (res) {
+ return res[1];
+ },
+ "yahoo": function (res) {
+ return _.map(res.gossip.results, 'key');
+ },
+ "ebay": function (res) {
+ return res.res ? res.res.sug : [];
+ },
+ "twitter": function (res) {
+ return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
+ },
+ "baidu": function (res) {
+ return res.s;
+ },
+ "yandex": function(res){
+ return _.map(res[1], function(r){
+ return typeof r === 'string' ? r : r[1];
+ });
+ },
+ };
+ var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"];
+ return parser(res);
+ },
+
+ setUpDb: function(success){
+ var self=this;
+ // setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB
+ var dbReq = window.indexedDB.open("KeywordShitter2", 3);
+ /** Error handler for all child transactions as events bubbe up **/
+ dbReq.onerror = function (event) {
+ return console.error('Error opening indexedDB database KeywordShitter2', event);
+ };
+ dbReq.onsuccess = function (event) {
+ self.db = event.target.result;
+ self.db.onerror = function (event) {
+ // Generic error handler requests
+ if (event && event.target && event.target.error)
+ console.error("Database error",event.target.error.errorCode, event.target.error.message, event);
+ else
+ console.error("Database error",arguments);
+ return this;
+ };
+ if (success) success(self.db,dbReq);
+ return self.db;
+ };
+ dbReq.onupgradeneeded = function (event) {
+ console.log("running onupgradeneeded");
+ self.db = event.target.result;
+
+ if (!self.db.objectStoreNames.contains("suggestions")) {
+ var objectStore = self.db.createObjectStore("suggestions", {
+ autoIncrement: true, keyPath: 'id'
+ });
+ // Create an index to search suggestions by
+ // he query that prompted the suggestion
+ if (!objectStore.indexNames.contains('search'))
+ objectStore.createIndex("search", "search", {unique: false});
+
+ // and by suggestion/keyword
+ if (!objectStore.indexNames.contains('keyword'))
+ objectStore.createIndex("keyword", "keyword", {unique: false});
+
+ }
+
+ return self.db;
+ };
+ return dbReq;
+ },
+
+ toggleWork: function(){
+ if (this.doWork === false)
+ this.StartWork();
else
- console.error("Database error",arguments);
- return this;
- };
- if (success) success(db,dbReq);
- return db;
- };
- dbReq.onupgradeneeded = function (event) {
- console.log("running onupgradeneeded");
- db = event.target.result;
+ this.StopWork();
+ },
- if (!db.objectStoreNames.contains("suggestions")) {
- var objectStore = db.createObjectStore("suggestions", {
- autoIncrement: true, keyPath: 'id'
- });
- // Create an index to search suggestions by
- // he query that prompted the suggestion
- if (!objectStore.indexNames.contains('search'))
- objectStore.createIndex("search", "search", {unique: false});
+ StartWork: function() {
+ if (this.doWork === false) {
+ // reset these
+ this.saveSettings();
+ $('#startjob').val('Stop Job').text('Stop shitting').addClass('btn-danger');
+ this.hashMapInputs = {};
+ this.keywordsToQuery = [];
+ this.keywordsToQueryIndex = 0;
- // and by suggestion/keyword
- if (!objectStore.indexNames.contains('keyword'))
- objectStore.createIndex("keyword", "keyword", {unique: false});
+ this.hashMapInputs[""] = true;
+ this.hashMapInputs[" "] = true;
+ this.hashMapInputs[" "] = true;
- }
+ // update config
+ this.options = this.getOptions();
- return db;
- };
- return dbReq;
-}
+ // get queries from the input
+ var ks = $('#input').val().split("\n");
+ this.keywordsToQuery=ks.map(this.CleanVal);
+ // add variations of the initial terms
+ // (before we start adding variations of the results)
+ if (!this.keywordsToQuery.length)
+ this.permuteResultsToQueue([' ']);
+ else {
+ var untickedInputs = this.keywordsToQuery.filter(function(k){
+ return k.slice(-1)!=='✓' && k.slice(-1)!=='❌';
+ });
+ this.permuteResultsToQueue(untickedInputs);
+ }
-function StartJob() {
- if (doWork === false) {
- hashMapInputs = {};
- keywordsToQuery = [];
- keywordsToQueryIndex = 0;
+ this.numOfInitialKeywords = this.keywordsToQuery.length;
+ // show the extended queue
+ this.FilterAndDisplay();
- hashMapInputs[""] = 1;
- // hashMapInputs[" "] = 1;
- hashMapInputs[" "] = 1;
+ this.doWork = true;
+ // $('#input').hide();
- // update config
- prefixes = $('#prefixes').val().split(',');
- suffixes = $('#suffixes').val().split(',');
-
- // get queries from the input
- var ks = $('#input').val().split("\n");
- for (var i = 0; i < ks.length; i++) {
- if (ks[i].trim().length)
- keywordsToQuery[keywordsToQuery.length] = ks[i];
- }
- numOfInitialKeywords = keywordsToQuery.length;
-
- // add variations of the initial terms
- // (before we start adding variations of the results)
- if (!numOfInitialKeywords)
- permuteResultsToQueue([' ']);
- else
- permuteResultsToQueue(keywordsToQuery);
- FilterAndDisplay();
-
- doWork = true;
- $('#startjob').val('Stop Job').text('Stop shitting').addClass('btn-danger');
- // $('#input').hide();
-
- } else {
- doWork = false;
- $('#startjob').val('Start Job').text('Start shitting').removeClass('btn-danger');
- // $('#input').show();
- FilterAndDisplay();
- table.draw();
- table.columns.adjust();
- saveSettings();
- }
-}
-
-function DoJob() {
- if (doWork === true && queryLock === false) {
- if (keywordsToQueryIndex < keywordsToQuery.length) {
- var currentKw = keywordsToQuery[keywordsToQueryIndex];
- if (currentKw[currentKw.length - 1] != '✓') {
- QueryKeyword(currentKw);
- keywordsToQueryIndex++;
} else {
- // we didn't do a query immediatly go to next query
- keywordsToQueryIndex++;
- DoJob();
- }
- } else {
- if (numOfInitialKeywords != keywordsToQuery.length) {
- doWork = false;
+ }
+ },
+
+ StopWork: function(){
+ if (this.doWork){
$('#startjob').val('Start Job').text('Start shitting').removeClass('btn-danger');
- $('#input').show();
- FilterAndDisplay();
- table.draw();
- table.columns.adjust();
- } else {
- keywordsToQueryIndex = 0;
+ this.doWork = false;
+ // $('#input').show();
+ this.table.draw();
+ this.table.columns.adjust();
+ this.saveSettings();
+ this.FilterAndDisplay();
}
- }
- }
-}
+ },
-/** Make permutations of results and add to queue **/
-function permuteResultsToQueue(retList, search){
- var chr, currentx, currentKw;
+ DoJob: function() {
+ if (this.doWork === true && this.queryLock === false) {
+ if (this.keywordsToQueryIndex < this.numOfInitialKeywords) {
+ var currentKw = this.keywordsToQuery[this.keywordsToQueryIndex];
+ if (currentKw.slice(-1)!=='✓' && currentKw.slice(-1)!=='❌') {
+ this.QueryKeyword(currentKw);
+ this.keywordsToQueryIndex++;
+ } else {
+ // we didn't do a query immediatly go to next query
+ this.keywordsToQueryIndex++;
+ this.DoJob();
+ }
- // sort so the shortest is first in the queue
- retList.sort(function (a, b) {
- return a.length - b.length;
- });
-
- // add each result to list first before permutations
- for (var j = 0; j < retList.length; j++) {
- cleanKw = CleanVal(retList[j]);
- // add base suggestion to queue if it's not already done and isn't empty
- if (cleanKw && cleanKw.trim().length && cleanKw!==search && !hashMapInputs[cleanKw])
- keywordsToQuery[keywordsToQuery.length] = cleanKw;
- hashMapInputs[cleanKw] = 1;
- }
- for (var i = 0; i < retList.length; i++) {
- cleanKw = CleanVal(retList[i]);
- if (cleanKw.length){
-
- // add prefix permutations
- for (var k = 0; k < prefixes.length; k++) {
- chr = prefixes[k];
- currentx = chr + ' ' + cleanKw;
- if (!hashMapInputs[currentx])
- keywordsToQuery[keywordsToQuery.length] = currentx;
- hashMapInputs[currentx] = 1;
+ } else {
+ if (this.options.keepRunning) {
+ console.log('finish initial queue');
+ this.StopWork();
+ this.StartWork();
+ } else {
+ console.log('finish initial queue');
+ this.StopWork();
+ }
+ }
}
- // add suffix permutations
- for (var j = 0; j < prefixes.length; j++) {
- chr = prefixes[j];
- currentx = cleanKw + ' ' + chr;
- if (!hashMapInputs[currentx])
- keywordsToQuery[keywordsToQuery.length] = currentx;
- hashMapInputs[currentx] = 1;
- }
- }
- }
-}
+ },
-/** export db as json **/
-function exportDB(success){
- var reqObj = db.transaction(["suggestions"],"readonly")
- .objectStore("suggestions")
- .getAll();
- reqObj.onsuccess = function(e) {
- var blob, name;
- if (e.target.result.length){
- var jsonData = JSON.stringify(e.target.result);
- blob = new Blob([jsonData], {type: "octet/stream"});
- var timeStamp = (new Date()).toISOString().replace(/[:\.]/g,'_').slice(0,-5);
- name = 'keywordshitter_'+timeStamp+'_r'+e.target.result.length+'.json';
- saveAs(blob,name);
- if (success) success(blob);
- }
- return blob;
- };
- return reqObj;
-}
+ addResultsToQueue: function(retList, search){
+ retList=retList.map(this.CleanVal);
-/** Display data from db upon pressing load button **/
-function loadFromDB(){
- var reqObj = db.transaction(["suggestions"],"readonly")
- .objectStore("suggestions")
- .getAll()
- .onsuccess = function(e) {
- if (e.target.result.length){
- /// grab the fields we want
- var data =[];
- for (var i = 0; i < e.target.result.length; i++) {
- var d = e.target.result[i];
- var da = [
- d.id,
- d.keyword,
- d.length,
- d.volume,
- d.cpc,
- d.search,
- d.domain
+ // add each result to list first before permutations
+ for (var j = 0; j < retList.length; j++) {
+ cleanKw = retList[j];
+ // add base suggestion to queue if it's not already done and isn't empty
+ if (cleanKw && cleanKw.length && !this.hashMapInputs[cleanKw] && this.keywordsToQuery.indexOf(cleanKw)===-1)
+ this.keywordsToQuery.push(cleanKw);
+ this.hashMapInputs[cleanKw] = true;
+ }
+
+ },
+
+ /** Make permutations of results and add to queue **/
+ permuteResultsToQueue: function(retList, search){
+ var chr, currentx, currentKw;
+ var self = this;
+
+ this.hashMapInputs[search] = true;
+
+ // sort so the shortest is first in the queue
+ // retList.sort(function (a, b) {
+ // return a.length - b.length;
+ // });
+
+ function addPrefix(s,prefix){
+ return prefix+' '+s;
+ }
+ function addSuffix(s,suffix){
+ return s+' '+suffix;
+ }
+ // clean
+ retList=retList.map(this.CleanVal);
+
+
+
+
+
+ // get permutations
+ var newInputs = retList.reduce(function(result, keyword){
+ return _.concat(
+ result,
+ self.options.prefixes.map(addPrefix.bind(self,keyword)),
+ self.options.suffixes.map(addSuffix.bind(self,keyword))
+ );
+ }, []);
+
+ // add to queue
+ this.keywordsToQuery.push.apply(this.keywordsToQuery,newInputs);
+
+ return newInputs;
+ },
+
+ /** export db as json **/
+ exportDB: function(success){
+ var reqObj = this.db.transaction(["suggestions"],"readonly")
+ .objectStore("suggestions")
+ .getAll();
+ reqObj.onsuccess = function(e) {
+ var blob, name;
+ if (e.target.result.length){
+ var jsonData = JSON.stringify(e.target.result);
+ blob = new Blob([jsonData], {type: "octet/stream"});
+ var timeStamp = (new Date()).toISOString().replace(/[:\.]/g,'_').slice(0,-5);
+ name = 'keywordshitter_'+timeStamp+'_r'+e.target.result.length+'.json';
+ saveAs(blob,name);
+ if (success) success(blob);
+ }
+ return blob;
+ };
+ return reqObj;
+ },
+
+ /** Display data from db upon pressing load button **/
+ loadFromDB: function(){
+ var self = this;
+ var reqObj = this.db.transaction(["suggestions"],"readonly")
+ .objectStore("suggestions")
+ .getAll()
+ .onsuccess = function(e) {
+ if (e.target.result.length){
+ /// grab the fields we want
+ var data =[];
+ for (var i = 0; i < e.target.result.length; i++) {
+ var d = e.target.result[i];
+ var da = [
+ d.id,
+ d.keyword,
+ d.length,
+ d.volume,
+ d.cpc,
+ d.search,
+ d.domain
+ ];
+
+ // also remove undefined so datatables doesn't bring up alerts
+ da = da.map(function(v){return v===undefined ? null: v;});
+
+ // parse nums
+ // da = da.map(v => /^[\d+\.]+$/.test(v) ? Number(v): v);
+ //
+ data.push(da);
+ }
+ self.table.rows.add(data);
+ return self.table.draw(false);
+ }
+ };
+ return;
+ },
+
+ /** Display results **/
+ displayResults: function(retList, search, dontDisplay, url,data){
+
+ var rows=[];
+ retList=retList.map(this.CleanVal);
+ for (var i = 0; i < retList.length; i++) {
+ var cleanKw = retList[i];
+
+ // url might be in retlist
+ if (url===undefined) url=data[i].url;
+
+ var da = [
+ this.table.rows()[0].length+i,
+ cleanKw,
+ cleanKw.length,
+ null,
+ null,
+ search,
+ this.extractDomain(url)
];
- // also remove undefined so datatables doesn't bring up alerts
- da = da.map(function(v){return v===undefined ? null: v;});
+ // remove undefined values to avoid datatable alerts
+ da = da.map(function(v){return v===undefined ? null: v;});
- // parse nums
- // da = da.map(v => /^[\d+\.]+$/.test(v) ? Number(v): v);
- //
- data.push(da);
- }
- table.rows.add(data);
- return table.draw(false);
+ // TODO Check if suggestion is already displayed before adding
+ // var matches = table.data().filter(function(v){return v[1]===cleanKw && v[5]==search;}).count();
+ // if (!matches)
+ rows.push(da);
}
- };
- return;
-}
+ this.table.rows.add(rows);
+ if (!dontDisplay) this.table.draw(false);
+ },
-/** Display results **/
-function displayResults(retList, search, dontDisplay, url,data){
-
- var rows=[];
- for (var i = 0; i < retList.length; i++) {
- var cleanKw = CleanVal(retList[i]);
-
- // url might be in retlist
- if (url===undefined) url=data[i].url;
-
- var da = [
- table.rows()[0].length+i,
- cleanKw,
- cleanKw.length,
- null,
- null,
- search,
- extractDomain(url)
- ];
-
- // remove undefined values to avoid datatable alerts
- da = da.map(function(v){return v===undefined ? null: v;});
-
- // TODO Check if suggestion is already displayed before adding
- // var matches = table.data().filter(function(v){return v[1]===cleanKw && v[5]==search;}).count();
- // if (!matches)
- rows.push(da);
- }
- table.rows.add(rows);
- if (!dontDisplay) table.draw(false);
-}
-
-/** Takes url string and returns domain e.g. www.google.com or google.com
- * and some extra params to identify is
- **/
-function extractDomain(url) {
- if (url===undefined) return null;
- var domain;
- //find & remove protocol (http, ftp, etc.) and get domain
- if (url.indexOf("://") > -1) {
- domain = url.split('/')[2];
- }
- else {
- domain = url.split('/')[0];
- }
-
- //find & remove port number
- domain = domain.split(':')[0];
-
- // custom, add ds= param to distinguish googles etc diff searchs
- var mr = url.match('ds=(..?)&');
- if (mr && mr[1] && mr[1].length) domain+='&ds='+mr[1];
-
- var mr = url.match('gl=(..?)&');
- if (mr && mr[1] && mr[1].length) domain+='&gl='+mr[1];
-
- var mr = url.match('hl=(..?)&');
- if (mr && mr[1] && mr[1].length) domain+='&hl='+mr[1];
-
- // lang code for yandex
- var mr = url.match('uil=(..?)&');
- if (mr && mr[1] && mr[1].length) domain+='&uil='+mr[1];
-
- return domain;
-}
-
-/** Store new results in db and hashmap **/
-function storeResults(retList, search, url){
-
- // We will add the items async in order
- // Ref: http://stackoverflow.com/a/13666741/221742
- var transaction = db.transaction(["suggestions"], "readwrite");
- var store = transaction.objectStore("suggestions");
- var i=0;
- addNext();
-
- /** Like an async for loop to add each to db **/
- function addNext() {
- if (i
-1) {
+ domain = url.split('/')[2];
}
else {
- // search not done, lets do the query
- url = getUrl()+search;
- $.ajax({
- url: url,
- jsonp: "jsonp",
- dataType: "jsonp",
- success: function (res, statusText, jqXHR) {
-
- var retList = parseServiceResponse(res);
- var char, currentx;
- if (retList && retList.length){
- storeResults(retList, search, this.url);
- displayResults(retList, search, undefined, this.url);
- permuteResultsToQueue(retList);
- } else {
- console.warn('No results for query "',search,'" ');
- }
- markAsDone(search);
- queryLock = false;
-
- },
- error: function(){
- queryLock = false;
- }
- });
+ domain = url.split('/')[0];
}
- };
-}
-/** Clean input, may not all be needed **/
-function CleanVal(input) {
- var val = input;
+ //find & remove port number
+ domain = domain.split(':')[0];
- // legacy
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("", "");
- // val = val.replace("&", "&");
- // val = val.replace("", "");
- // val = val.replace("&", "");
- // val = val.replace("'", "'");
- // val = val.replace("#39;", "'");
- // val = val.replace("", "");
- // val = val.replace("–", "2013");
+ // custom, add ds= param to distinguish googles etc diff searchs
+ var mr = url.match('ds=(..?)&');
+ if (mr && mr[1] && mr[1].length) domain+='&ds='+mr[1];
- // this removes navigation suggestions
- if (val.length > 4 && val.substring(0, 4) == "http") val = "";
- return val;
-}
+ var mr = url.match('gl=(..?)&');
+ if (mr && mr[1] && mr[1].length) domain+='&gl='+mr[1];
-/** TODO get this working **/
-function Filter(listToFilter) {
- var retList = listToFilter;
+ var mr = url.match('hl=(..?)&');
+ if (mr && mr[1] && mr[1].length) domain+='&hl='+mr[1];
- if ($("#filter-positive").val().length > 0) {
- var filteredList = [];
- var filterContains = $("#filter-positive").val().split("\n");
- for (var i = 0; i < retList.length; i++) {
- var currentKeyword = retList[i];
- var boolContainsKeyword = false;
- for (var j = 0; j < filterContains.length; j++) {
- if (filterContains[j].length > 0) {
- if (currentKeyword.indexOf(filterContains[j]) != -1) {
- boolContainsKeyword = true;
- break;
+ // lang code for yandex
+ var mr = url.match('uil=(..?)&');
+ if (mr && mr[1] && mr[1].length) domain+='&uil='+mr[1];
+
+ return domain;
+ },
+
+ /** Store new results in db and hashmap **/
+ storeResults: function(retList, search, url){
+ var self = this;
+ retList=retList.map(this.CleanVal);
+
+ // We will add the items async in order
+ // Ref: http://stackoverflow.com/a/13666741/221742
+ var transaction = this.db.transaction(["suggestions"], "readwrite");
+ var store = transaction.objectStore("suggestions");
+ var i=0;
+ addNext();
+
+ /** Like an async for loop to add each to db **/
+ function addNext() {
+ if (i 0) {
- var filteredList = [];
- var filterContains = $("#filter-negative").val().split("\n");
- for (var i = 0; i < retList.length; i++) {
- var currentKeyword = retList[i];
- var boolCleanKeyword = true;
- for (var j = 0; j < filterContains.length; j++) {
- if (filterContains[j].length > 0) {
- if (currentKeyword.indexOf(filterContains[j]) >= 0) {
- boolCleanKeyword = false;
- break;
+ /** mark a search as done in the queue **/
+ markAsNone: function(search){
+ // mark as done in queue
+ if (this.keywordsToQuery[this.keywordsToQueryIndex]===search)
+ this.keywordsToQuery[this.keywordsToQueryIndex]+=' ❌';
+ else if (this.keywordsToQuery[this.keywordsToQueryIndex-1]===search)
+ this.keywordsToQuery[this.keywordsToQueryIndex-1]+=' ❌';
+ else
+ console.warn('Cant find ',search,'in keywordsToQuery');
+ },
+
+ /** Get search suggestions for a keyword **/
+ QueryKeyword: function(search) {
+ var self = this;
+ this.queryLock = true;
+
+ // first check in db
+ var reqObj = this.db.transaction(["suggestions"],"readonly")
+ .objectStore("suggestions")
+ .index("search")
+ .getAll(search);
+ reqObj.onsuccess = function(e) {
+ var domain = self.extractDomain(self.getUrl());
+ var results = e.target.result.filter(function(r){return r.domain==domain;});
+ // console.log(e.target.result);
+ if (results.length){
+ // search was done previously so display results from db
+ var retList=_.map(results,'keyword');
+ self.markAsDone(search);
+ self.displayResults(retList,search,undefined,undefined,results);
+ self.addResultsToQueue(retList);
+ if (self.options.keepRunning) self.permuteResultsToQueue(retList);
+ self.queryLock = false;
}
- }
- }
+ else {
+ // search not done, lets do the query
+ url = self.getUrl()+search;
+ $.ajax({
+ url: url,
+ jsonp: "jsonp",
+ dataType: "jsonp",
+ success: function (res, statusText, jqXHR) {
- if (boolCleanKeyword) {
- filteredList[filteredList.length] = currentKeyword;
- }
- }
+ var retList = self.parseServiceResponse(res);
+ var char, currentx;
+ if (retList && retList.length){
+ self.storeResults(retList, search, this.url);
+ self.displayResults(retList, search, undefined, this.url);
+ self.addResultsToQueue(retList);
+ if (self.options.keepRunning) self.permuteResultsToQueue(retList);
+ self.markAsDone(search);
+ } else {
+ // console.debug('No suggestions for query: "',search,'"');
+ self.markAsNone(search);
+ }
+ self.queryLock = false;
+ return;
- retList = filteredList;
- }
+ },
+ error: function(e){
+ console.error(e);
+ self.queryLock = false;
+ return;
+ }
+ });
+ }
+ };
+ return reqObj;
+ },
- return retList;
-}
+ /** Clean input, may not all be needed **/
+ CleanVal: function(input) {
-/** display the queue, and update description of it **/
-function FilterAndDisplay() {
- var i = 0;
- var sb = '';
-
- var outputKeywords = keywordsToQuery;
- for (i = 0; i < Math.min(outputKeywords.length,maxQueueDisplay); i++) {
- sb += outputKeywords[i];
- sb += '\n';
- }
- if (outputKeywords.length>maxQueueDisplay) sb+='...\n';
- $("#input").val(sb);
- $("#numofkeywords").html('Queue:' + outputKeywords.length);
-}
-
-
-/** overrides default with dom options with arguments options **/
-function getOptions(argOptions){
- var defaultOptions={}; // for now defaults are set in html
- if (argOptions===undefined) argOptions={};
- return _.defaults(argOptions,getDomOptions(),defaultOptions);
-}
-
-/** read settings from webpage **/
-function getDomOptions(){
- return {
- service : $('#service').val(),
- filterNegative: $('#filter-negative').val(),
- filterPositive: $('#filter-positive').val(),
- rateLimit: $('#rate-limit').val(),
- // input: $('#input').val(),
- prefixes: $('#prefixes').val(),
- suffixes: $('#suffixes').val(),
- country: $('#country').val(),
- lang: $('#lang').val(),
- };
-}
-
-/** load settings from localStorage **/
-function loadSettings(){
- // TODO do table settings as well, e.g. column visibilitity
- if (localStorage.service) $("#service").val( localStorage.service );
- if (localStorage.country) $('#country').val(localStorage.country);
- if (localStorage.lang) $('#lang').val(localStorage.lang);
- if (localStorage.filterNegative) $("#filter-negative").val( localStorage.filterNegative );
- if (localStorage.filterPositive) $("#filter-positive").val( localStorage.filterPositive );
- if (localStorage.rateLimit) $("#rate-limit").val( localStorage.rateLimit );
- if (localStorage.input) $("#input").val( localStorage.input );
- if (localStorage.prefixes) $("#prefixes").val( localStorage.prefixes );
- if (localStorage.suffixes) $("#suffixes").val( localStorage.suffixes );
-
-}
-/** save settings to localStorage. **/
-function saveSettings(){
- localStorage.service = $('#service').val();
- localStorage.country = $('#country').val();
- localStorage.lang = $('#lang').val();
- localStorage.filterNegative = $('#filter-negative').val();
- localStorage.filterPositive = $('#filter-positive').val();
- localStorage.rateLimit = $('#rate-limit').val();
- localStorage.input = $('#input').val();
- localStorage.prefixes = $('#prefixes').val();
- localStorage.suffixes = $('#suffixes').val();
-}
-
-/** reset inputs and results, but not settings **/
-function reset(){
- table.clear();
- table.draw();
- $('#input').val('');
- saveSettings();
-}
-
-function kws2Init(){
- setUpDb();
-
- window.setInterval(DoJob, 750);
- //
- // $('#startjob').on('click',StartJob);
- // $('#reset').on('click',reset);
- // $('#load-from-cache').on('click',loadFromDB);
-
- loadSettings();
- table = $('#outtable').DataTable({
- pageLength: 50,
- "lengthMenu": [ 10, 25, 50, 75, 100,800],
- dom:
- "<'row'<'col-sm-5'B><'col-sm-7'<'pull-right'p>>>" +
- "<'row'<'col-sm-8'i><'col-sm-4'<'pull-right'f>>>" +
- "<'row'<'col-sm-12'tr>>",
- buttons: [
- 'colvis',
- 'pageLength',
- {
- extend: 'collection',
- text: 'Export',
- buttons: [
- 'copyHtml5',
- 'csvHtml5',
- ]
+ function escapeHtml(unsafe) {
+ return unsafe
+ .replace(/&/g, "&")
+ .replace(//g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
}
- // {
- // extend: 'csvHtml5',
- // fieldBoundary: "",
- // text: 'Copy keywords',
- // // 'customize': function(data,options){return data.split('\n').join(',');},
- // header: false,
- // exportOptions: {
- // stripNewlines: true,
- // stripHtml: true,
- // decodeEntities: true,
- // columns: 1,
- // // format:{
- // // body: function(html,i){console.log(html);return html}
- // // }
- // }
- // },
- ],
- "columnDefs": [
- {
- "name": "id",
- "targets": 0,
- "visible": false,
- }, {
- "responsivePriority": 1,
- "name": "keyword",
- "targets": 1
- }, {
- "name": "length",
- "targets": 2,
- "visible": false,
- }, {
- "name": "volume",
- "targets": 3,
- "visible": false,
- }, {
- "name": "cpc",
- "targets": 4,
- "visible": false,
- }, {
- "name": "search",
- "responsivePriority": 3,
- "targets": 5,
- "visible": false,
- }, {
- "name": "domain",
- "responsivePriority": 2,
- "targets": 6,
- "visible": false,
- }],
- order: [[ 0, 'desc' ]],
- // colReorder: {},
- stateSave: true,
- fixedHeader: true,
- // responsive: true,
- // scrollY: 500,
- // deferRender: true,
- // scroller: true
- });
- $.getJSON('https://api.ipify.org?format=json', function (data) {
- myIp = data.ip;
- });
+ // google returns lowercase
+ input = input.toLowerCase();
+ input = escapeHtml(input);
-}
+ // not sure if we want to trim, the search engines are sensitive to
+ // whitespace. (and also last search)
+ // input = input.trim();
+
+ // this removes navigation suggestions
+ if (input.length > 4 && input.substring(0, 4) == "http") input = "";
+ return input;
+ },
+
+ /** TODO get this working **/
+ Filter: function(listToFilter) {
+ var retList = listToFilter;
+
+ if ($("#filter-positive").val().length > 0) {
+ var filteredList = [];
+ var filterContains = $("#filter-positive").val().split("\n");
+ for (var i = 0; i < retList.length; i++) {
+ var currentKeyword = retList[i];
+ var boolContainsKeyword = false;
+ for (var j = 0; j < filterContains.length; j++) {
+ if (filterContains[j].length > 0) {
+ if (currentKeyword.indexOf(filterContains[j]) != -1) {
+ boolContainsKeyword = true;
+ break;
+ }
+ }
+ }
+
+ if (boolContainsKeyword) {
+ filteredList[filteredList.length] = currentKeyword;
+ }
+ }
+
+ retList = filteredList;
+ }
+
+ if ($("#filter-negative").val().length > 0) {
+ var filteredList = [];
+ var filterContains = $("#filter-negative").val().split("\n");
+ for (var l = 0; l < retList.length; l++) {
+ var currentKeyword = retList[l];
+ var boolCleanKeyword = true;
+ for (var k = 0; k < filterContains.length; k++) {
+ if (filterContains[k].length > 0) {
+ if (currentKeyword.indexOf(filterContains[k]) >= 0) {
+ boolCleanKeyword = false;
+ break;
+ }
+ }
+ }
+
+ if (boolCleanKeyword) {
+ filteredList[filteredList.length] = currentKeyword;
+ }
+ }
+
+ retList = filteredList;
+ }
+
+ return retList;
+ },
+
+ /** display the queue, and update description of it **/
+ FilterAndDisplay: function() {
+ var i = 0;
+ var sb = '';
+
+ var outputKeywords = this.keywordsToQuery;
+ for (i = 0; i < Math.min(outputKeywords.length,this.options.maxQueueDisplay); i++) {
+ sb += outputKeywords[i];
+ sb += '\n';
+ }
+ if (outputKeywords.length>this.options.maxQueueDisplay) sb+='...\n';
+ $("#input").val(sb);
+ $("#numofkeywords").html('Queue:' + outputKeywords.length);
+ },
+
+
+ /** overrides default with dom options with arguments options **/
+ getOptions: function(argOptions){
+ var defaultOptions={
+ keepRunning: false,
+ maxQueueDisplay: 10000,
+ country: "",
+ filterNegative: "",
+ filterPositive: "",
+ lang: "",
+ prefixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "how", "which", "why", "where", "who", "when", "are", "what"],
+ rateLimit: 750,
+ service: "google",
+ suffixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "like", "for", "without", "with", "versus", "vs", "to", "near", "except", "has"]
+ }; // for now defaults are set in html
+ if (argOptions===undefined) argOptions={};
+ return _.defaults(argOptions,this.getDomOptions(),defaultOptions);
+ },
+
+ /** read settings from webpage **/
+ getDomOptions: function(){
+
+ var service= $('#service').val(),
+ filterNegative = $('#filter-negative').val(),
+ filterPositive = $('#filter-positive').val(),
+ rateLimit = parseInt($('#rate-limit').val()),
+ // input: $('#input').val(),
+ prefixes = $('#prefixes').val(),
+ suffixes = $('#suffixes').val(),
+ country = $('#country').val(),
+ lang = $('#lang').val(),
+ keepRunning = $('#keep-running').prop('checked');
+ if (prefixes && prefixes.length)
+ prefixes=prefixes.split(',');
+ else
+ prefixes=undefined;
+ if (suffixes && suffixes.length)
+ suffixes=suffixes.split(',');
+ else
+ suffixes=undefined;
+
+ var options={};
+ if (service) options.service=service;
+ if (filterNegative) options.filterNegative=filterNegative;
+ if (filterPositive) ooptions.filterPositive=filterPositive;
+ if (rateLimit) options.rateLimit=rateLimit;
+ if (prefixes) options.prefixes=prefixes;
+ if (suffixes) options.suffixes=suffixes;
+ if (country) options.country=country;
+ if (lang) options.lang=lang;
+ if (keepRunning) options.keepRunning=keepRunning;
+ return options;
+ },
+
+ /** load settings from localStorage **/
+ loadSettings: function(){
+ // TODO do table settings as well, e.g. column visibilitity
+ if (localStorage.service) $("#service").val( localStorage.service );
+ if (localStorage.country) $('#country').val(localStorage.country);
+ if (localStorage.lang) $('#lang').val(localStorage.lang);
+ if (localStorage.filterNegative) $("#filter-negative").val( localStorage.filterNegative );
+ if (localStorage.filterPositive) $("#filter-positive").val( localStorage.filterPositive );
+ if (localStorage.rateLimit) $("#rate-limit").val( localStorage.rateLimit );
+ if (localStorage.input) $("#input").val( localStorage.input );
+ if (localStorage.prefixes) $("#prefixes").val( localStorage.prefixes );
+ if (localStorage.suffixes) $("#suffixes").val( localStorage.suffixes );
+ if (localStorage.keepRunning) $('#keep-running').prop('checked',localStorage.keepRunning=="true");
+
+ },
+ /** save settings to localStorage. **/
+ saveSettings: function(){
+ localStorage.service = $('#service').val();
+ localStorage.country = $('#country').val();
+ localStorage.lang = $('#lang').val();
+ localStorage.filterNegative = $('#filter-negative').val();
+ localStorage.filterPositive = $('#filter-positive').val();
+ localStorage.rateLimit = $('#rate-limit').val();
+ localStorage.input = $('#input').val();
+ localStorage.prefixes = $('#prefixes').val();
+ localStorage.suffixes = $('#suffixes').val();
+ localStorage.keepRunning = $('#keep-running').prop('checked');
+ },
+
+ /** reset inputs and results, but not settings **/
+ reset: function(){
+ this.table.clear();
+ this.table.draw();
+ $('#input').val('');
+ this.saveSettings();
+ },
+
+ init: function(){
+ this.setUpDb();
+ this.loadSettings();
+ this.options = this.getOptions();
+
+ window.setInterval(this.DoJob.bind(this), this.options.rateLimit);
+
+ $('#startjob').on('click',this.toggleWork.bind(this));
+ $('#reset').on('click',this.reset.bind(this));
+ $('#load-from-cache').on('click',this.loadFromDB.bind(this));
+ $('#export-from-cache').on('click',this.exportDB.bind(this));
+ // $('#filter-positive').on('click',this.FilterIfNotWorking.bind(this));
+ // $('#filter-negative').on('click',this.FilterIfNotWorking.bind(this));
+
+
+ this.table = $('#outtable').DataTable({
+ pageLength: 50,
+ "lengthMenu": [ 10, 25, 50, 75, 100,800],
+ dom:
+ "<'row'<'col-sm-5'B><'col-sm-7'<'pull-right'p>>>" +
+ "<'row'<'col-sm-8'i><'col-sm-4'<'pull-right'f>>>" +
+ "<'row'<'col-sm-12'tr>>",
+ buttons: [
+ 'colvis',
+ 'pageLength',
+ {
+ extend: 'collection',
+ text: 'Export',
+ buttons: [
+ 'copyHtml5',
+ 'csvHtml5',
+ ]
+ }
+ // {
+ // extend: 'csvHtml5',
+ // fieldBoundary: "",
+ // text: 'Copy keywords',
+ // // 'customize': function(data,options){return data.split('\n').join(',');},
+ // header: false,
+ // exportOptions: {
+ // stripNewlines: true,
+ // stripHtml: true,
+ // decodeEntities: true,
+ // columns: 1,
+ // // format:{
+ // // body: function(html,i){console.log(html);return html}
+ // // }
+ // }
+ // },
+ ],
+ "columnDefs": [
+ {
+ "name": "id",
+ "targets": 0,
+ "visible": false,
+ }, {
+ "responsivePriority": 1,
+ "name": "keyword",
+ "targets": 1
+ }, {
+ "name": "length",
+ "targets": 2,
+ "visible": false,
+ }, {
+ "name": "volume",
+ "targets": 3,
+ "visible": false,
+ }, {
+ "name": "cpc",
+ "targets": 4,
+ "visible": false,
+ }, {
+ "name": "search",
+ "responsivePriority": 3,
+ "targets": 5,
+ "visible": false,
+ }, {
+ "name": "domain",
+ "responsivePriority": 2,
+ "targets": 6,
+ "visible": false,
+ }],
+ order: [[ 0, 'desc' ]],
+ // colReorder: {},
+ stateSave: true,
+ fixedHeader: true,
+ // responsive: true,
+ // scrollY: 500,
+ // deferRender: true,
+ // scroller: true
+ });
+
+ $.getJSON('https://api.ipify.org?format=json', function (data) {
+ this.myIp = data.ip;
+ });
+
+ }
+ };
+}();
diff --git a/public/js/test.js b/public/js/test.js
index 59a260e..e27081a 100644
--- a/public/js/test.js
+++ b/public/js/test.js
@@ -1,10 +1,29 @@
var expect = chai.expect;
var assert = chai.assert;
-// TODO describe('permute', function(){
-// it('should correctly permute results and add to queue', function() {
-// });
-// });
+// TODO
+describe('permute', function(){
+ before(function() {
+ KWS.hashMapInputs = {};
+ KWS.keywordsToQuery = [];
+ });
+
+ after(function() {
+ KWS.hashMapInputs = {};
+ KWS.keywordsToQuery = [];
+ });
+
+ it('should correctly permute results and add to queue', function() {
+ var retList = ['a','b','c d',' a longer one'];
+ KWS.permuteResultsToQueue(retList,'test');
+ assert(KWS.keywordsToQuery.length>0);
+ assert(KWS.hashMapInputs['b']);
+ assert(KWS.hashMapInputs['test']);
+ // var options = KWS.getOptions();
+ // var expectedLength = options.prefixes.length*retList.length+options.suffixes.length*retList.length
+ // assert(KWS.keywordsToQuery.length==expectedLength,''+expectedLength+'!='+KWS.keywordsToQuery.length);
+ });
+});
// TODO use localForage or most popular promise wrapper for indexedDB then test
// describe('exportDB', function(){
@@ -12,7 +31,7 @@ var assert = chai.assert;
describe('db', function() {
it('should correctly set up db', function(done) {
- var dbReq=setUpDb();
+ var dbReq=KWS.setUpDb();
var oldonsuccess=dbReq.onsuccess;
dbReq.onsuccess=function(event){
// override onld onsucess because it doesn't use promises
@@ -33,7 +52,7 @@ describe('extractDomain', function(){
if (testData.domainUrls.hasOwnProperty(expectedDomain)) {
it('should correctly extract domain from url, domain='+expectedDomain, function() {
var url = testData.domainUrls[expectedDomain];
- var domain = extractDomain(url);
+ var domain = KWS.extractDomain(url);
assert.equal(expectedDomain,domain);
});
}
@@ -48,7 +67,7 @@ describe('responses', function(){
it('should correctly parse test data for service='+service, function() {
var res = testData.responses[service];
var options = {service:service};
- var parsedRes = parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,options);
assert.typeOf(parsedRes,'array');
expect(parsedRes).to.have.length.above(1);
@@ -82,7 +101,7 @@ describe('services', function() {
searches.forEach(function(search){
it('should correctly get and parse data'+'. Service="'+service+'" search="'+search+'"', function(done) {
var options = {service:service};
- var url = getUrl(options)+search;
+ var url = KWS.getUrl(options)+search;
return $.ajax({
url: url,
async: false,
@@ -91,7 +110,7 @@ describe('services', function() {
success: function (res, statusText, jqXHR) {
assert(statusText=="success");
assert(res!==[]);
- var parsedRes = parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,options);
assert.typeOf(parsedRes,'array');
parsedRes.map(r => assert.typeOf(r, 'string'));
done();
@@ -110,7 +129,7 @@ describe('services', function() {
searchesDifficult.forEach(function(search){
it('should get and parse difficult data'+'. Service="'+service+'" search="'+search+'"', function(done) {
var options = {service:service};
- var url = getUrl(options)+search;
+ var url = KWS.getUrl(options)+search;
return $.ajax({
url: url,
async: false,
@@ -120,7 +139,7 @@ describe('services', function() {
assert(statusText=="success");
assert(res!==[]);
- var parsedRes = parseServiceResponse(res,options);
+ var parsedRes = KWS.parseServiceResponse(res,options);
assert.typeOf(parsedRes,'array');
parsedRes.map(r => assert.typeOf(r, 'string'));
return done();