diff --git a/index.html b/index.html index c377de7..e3360b8 100644 --- a/index.html +++ b/index.html @@ -64,10 +64,6 @@ lying internet

- Load cache - Export cache - - Delete cache
@@ -484,7 +480,7 @@ lying internet - + diff --git a/js/main.js b/js/main.js index fade330..42c5841 100644 --- a/js/main.js +++ b/js/main.js @@ -1,7 +1,6 @@ var KWS = function(){ return { - db:undefined, table: undefined, myIp: undefined, options: {}, @@ -13,132 +12,10 @@ var KWS = function(){ keywordsToQuery: [], keywordsToQueryIndex: 0, numOfInitialKeywords: 0, - services:{ - "google": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=", - "google news": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=", - "google shopping": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=", - "google books": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=", - "youtube": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=", - "google videos": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=", - "google images": - "http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=", - "yahoo": - "https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=", - "bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=", - "ebay": - "http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=", - "amazon": - "http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=", - "twitter": - "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=", - "baidu": "http://suggestion.baidu.com/su?cb=?&wd=", - "yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=", - "google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", // - "google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=", - "google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=", - "google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=", - // "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp - }, - /** - * Get the service url based on options set in the dom. - * @return {String} A jsonp url for search suggestions with query missing from the end. - */ - getUrl :function(){ - // Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js - // Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins - // Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list - // https://developers.google.com/custom-search/docs/ref_languages - return _.template(this.services[this.options.service])(this.options); - }, - - /** Parse response per service **/ - parseServiceResponse: function(res){ - // Each take a json response tand return a keyword array - RESPONSE_TEMPLATES = { - // opensearch default - "default": function (res) { - return res[1]; - }, - "yahoo": function (res) { - return _.map(res.gossip.results, 'key'); - }, - "ebay": function (res) { - return res.res ? res.res.sug : []; - }, - "twitter": function (res) { - return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick); - }, - "baidu": function (res) { - return res.s; - }, - "yandex": function(res){ - return _.map(res[1], function(r){ - return typeof r === 'string' ? r : r[1]; - }); - }, - "linkedin": function(res){ - return _.map(res.resultList,'displayName'); - }, - "google play": function(res){return _.map(res,'s')}, - "google play apps": function(res){return _.map(res,'s')}, - "google play movies": function(res){return _.map(res,'s')}, - "google play books": function(res){return _.map(res,'s')}, - }; - var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"]; - return parser(res); - }, - - setUpDb: function(success){ - var self=this; - // setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB - var dbReq = window.indexedDB.open("KeywordShitter2", 3); - /** Error handler for all child transactions as events bubbe up **/ - dbReq.onerror = function (event) { - return console.error('Error opening indexedDB database KeywordShitter2', event); - }; - dbReq.onsuccess = function (event) { - self.db = event.target.result; - self.db.onerror = function (event) { - // Generic error handler requests - if (event && event.target && event.target.error) - console.error("Database error",event.target.error.errorCode, event.target.error.message, event); - else - console.error("Database error",arguments); - return this; - }; - if (success) success(self.db,dbReq); - return self.db; - }; - dbReq.onupgradeneeded = function (event) { - console.log("running onupgradeneeded"); - self.db = event.target.result; - - if (!self.db.objectStoreNames.contains("suggestions")) { - var objectStore = self.db.createObjectStore("suggestions", { - autoIncrement: true, keyPath: 'id' - }); - // Create an index to search suggestions by - // he query that prompted the suggestion - if (!objectStore.indexNames.contains('search')) - objectStore.createIndex("search", "search", {unique: false}); - - // and by suggestion/keyword - if (!objectStore.indexNames.contains('keyword')) - objectStore.createIndex("keyword", "keyword", {unique: false}); - - } - - return self.db; - }; - return dbReq; - }, + services:suggestions.services, + getUrl :suggestions.getUrl, + parseServiceResponse: suggestions.parseServiceResponse, toggleWork: function(){ if (this.doWork === false) @@ -165,7 +42,7 @@ var KWS = function(){ // get queries from the input var ks = $('#input').val().split("\n"); - this.keywordsToQuery=ks.map(this.CleanVal); + this.keywordsToQuery=_.map(ks,this.CleanVal); // add variations of the initial terms @@ -238,7 +115,7 @@ var KWS = function(){ }, addResultsToQueue: function(retList, search){ - retList=retList.map(this.CleanVal); + retList=_.map(retList,this.CleanVal); // add each result to list first before permutations for (var j = 0; j < retList.length; j++) { @@ -255,6 +132,7 @@ var KWS = function(){ permuteResultsToQueue: function(retList, search){ var chr, currentx, currentKw; var self = this; + var options = this.getOptions() this.hashMapInputs[search] = true; @@ -270,14 +148,14 @@ var KWS = function(){ return s+' '+suffix; } // clean - retList=retList.map(this.CleanVal); + retList=_.map(retList,this.CleanVal); // get permutations var newInputs = retList.reduce(function(result, keyword){ return _.concat( result, - self.options.prefixes.map(addPrefix.bind(self,keyword)), - self.options.suffixes.map(addSuffix.bind(self,keyword)) + _.map(options.prefixes,addPrefix.bind(self,keyword)), + _.map(options.suffixes,addSuffix.bind(self,keyword)) ); }, []); @@ -287,66 +165,12 @@ var KWS = function(){ return newInputs; }, - /** export db as json **/ - exportDB: function(success){ - var reqObj = this.db.transaction(["suggestions"],"readonly") - .objectStore("suggestions") - .getAll(); - reqObj.onsuccess = function(e) { - var blob, name; - if (e.target.result.length){ - var jsonData = JSON.stringify(e.target.result); - blob = new Blob([jsonData], {type: "octet/stream"}); - var timeStamp = (new Date()).toISOString().replace(/[:\.]/g,'_').slice(0,-5); - name = 'keywordshitter_'+timeStamp+'_r'+e.target.result.length+'.json'; - saveAs(blob,name); - if (success instanceof Function) success(blob); - } - return blob; - }; - return reqObj; - }, - - clearDB: function(){ - this.db.transaction(["suggestions"], "readwrite") - .objectStore("suggestions") - .clear(); - console.warn('cleared all indexedDB data'); - }, - - /** Display data from db upon pressing load button **/ - loadFromDB: function(){ - var self = this; - var reqObj = this.db.transaction(["suggestions"],"readonly") - .objectStore("suggestions") - .getAll() - .onsuccess = function(e) { - if (e.target.result.length){ - /// grab the fields we want - var data =[]; - for (var i = 0; i < e.target.result.length; i++) { - var da = e.target.result[i]; - - // also remove undefined so datatables doesn't bring up alerts - da = _.mapValues(da, function(v){return v===undefined ? null: v;}); - - // parse nums - // da = da.map(v => /^[\d+\.]+$/.test(v) ? Number(v): v); - // - data.push(da); - } - self.table.rows.add(data); - return self.table.draw(false); - } - }; - return; - }, /** Display results **/ displayResults: function(retList, search, dontDisplay, url,data){ var rows=[]; - retList=retList.map(this.CleanVal); + retList=_.map(retList,this.CleanVal); for (var i = 0; i < retList.length; i++) { var cleanKw = retList[i]; @@ -411,49 +235,6 @@ var KWS = function(){ return domain; }, - /** Store new results in db and hashmap **/ - storeResults: function(retList, search, url){ - var self = this; - retList=retList.map(this.CleanVal); - - // We will add the items async in order - // Ref: http://stackoverflow.com/a/13666741/221742 - var transaction = this.db.transaction(["suggestions"], "readwrite"); - var store = transaction.objectStore("suggestions"); - var i=0; - addNext(); - - /** Like an async for loop to add each to db **/ - function addNext() { - if (i0); - assert(KWS.hashMapInputs['b']); - assert(KWS.hashMapInputs['test']); + expect(KWS.keywordsToQuery).to.have.length.above(0) + // assert(KWS.hashMapInputs['b']); + expect(KWS.hashMapInputs).to.include.key('test'); // var options = KWS.getOptions(); // var expectedLength = options.prefixes.length*retList.length+options.suffixes.length*retList.length // assert(KWS.keywordsToQuery.length==expectedLength,''+expectedLength+'!='+KWS.keywordsToQuery.length); }); }); -// TODO use localForage or most popular promise wrapper for indexedDB then test -// describe('exportDB', function(){ -// }); - -describe('db', function() { - it('should correctly set up db', function(done) { - var dbReq=KWS.setUpDb(); - var oldonsuccess=dbReq.onsuccess; - dbReq.onsuccess=function(event){ - // override onld onsucess because it doesn't use promises - var db=oldonsuccess.bind(dbReq)(event); - assert(db); - assert(db.objectStoreNames.contains('suggestions'),'db should contain suggestons store'); - var indexNames =db.transaction('suggestions','readonly').objectStore('suggestions').indexNames; - assert(indexNames.contains('keyword','search')); - done(); - }; - }); -}); - describe('extractDomain', function(){ @@ -66,11 +46,10 @@ describe('responses', function(){ if (testData.responses.hasOwnProperty(service)) { it('should correctly parse test data for service='+service, function() { var res = testData.responses[service]; - var options = {service:service}; - var parsedRes = KWS.parseServiceResponse(res,options); + var parsedRes = KWS.parseServiceResponse(res,service); - assert.typeOf(parsedRes,'array'); - expect(parsedRes).to.have.length.above(1); + expect(parsedRes).to.be.an('array') + .that.has.length.above(1); parsedRes.map(r => assert.typeOf(r, 'string')); }); } @@ -101,8 +80,7 @@ describe('services', function() { services.forEach(function(service){ searches.forEach(function(search){ it('should correctly get and parse data'+'. Service="'+service+'" search="'+search+'"', function(done) { - var options = {service:service}; - var url = KWS.getUrl(options)+search; + var url = KWS.getUrl(service)+search; return $.ajax({ url: url, async: false, @@ -111,7 +89,7 @@ describe('services', function() { success: function (res, statusText, jqXHR) { assert(statusText=="success"); assert(res!==[]); - var parsedRes = KWS.parseServiceResponse(res,options); + var parsedRes = KWS.parseServiceResponse(res,service); assert.typeOf(parsedRes,'array'); parsedRes.map(r => assert.typeOf(r, 'string')); done(); @@ -129,8 +107,7 @@ describe('services', function() { services.forEach(function(service){ searchesDifficult.forEach(function(search){ it('should get and parse difficult data'+'. Service="'+service+'" search="'+search+'"', function(done) { - var options = {service:service}; - var url = KWS.getUrl(options)+search; + var url = KWS.getUrl(service)+search; return $.ajax({ url: url, async: false, @@ -140,7 +117,7 @@ describe('services', function() { assert(statusText=="success"); assert(res!==[]); - var parsedRes = KWS.parseServiceResponse(res,options); + var parsedRes = KWS.parseServiceResponse(res,service); assert.typeOf(parsedRes,'array'); parsedRes.map(r => assert.typeOf(r, 'string')); return done(); diff --git a/public/test.html b/public/test.html index c6b4782..e1584a0 100644 --- a/public/test.html +++ b/public/test.html @@ -25,6 +25,7 @@ +