mirror of
https://github.com/wassname/keywordshitter2.git
synced 2026-06-27 16:10:23 +08:00
Merge branch 'master' into gh-pages
This commit is contained in:
+1
-5
@@ -64,10 +64,6 @@ lying internet</textarea>
|
||||
<div class="col-sm-5">
|
||||
<form id="advanced" class="collapse">
|
||||
<br>
|
||||
<a class="btn btn-default" id="load-from-cache" title="Load cached data from previous sessions" type="button" >Load cache</a>
|
||||
<a class="btn btn-default" id="export-from-cache" title="Export cached data from previous sessions as a json file" type="button">Export cache</a>
|
||||
|
||||
<a class="btn btn-default" id="clear-cache" title="Delete cached data" type="button">Delete cache</a>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="service">Service:</label>
|
||||
@@ -484,7 +480,7 @@ lying internet</textarea>
|
||||
|
||||
|
||||
|
||||
|
||||
<script type="text/javascript" src="js/suggestions.js"></script>
|
||||
<script type="text/javascript" src="js/main.js"></script>
|
||||
|
||||
|
||||
|
||||
+65
-293
@@ -1,7 +1,6 @@
|
||||
var KWS = function(){
|
||||
|
||||
return {
|
||||
db:undefined,
|
||||
table: undefined,
|
||||
myIp: undefined,
|
||||
options: {},
|
||||
@@ -13,132 +12,10 @@ var KWS = function(){
|
||||
keywordsToQuery: [],
|
||||
keywordsToQueryIndex: 0,
|
||||
numOfInitialKeywords: 0,
|
||||
services:{
|
||||
"google":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
|
||||
"google news":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
|
||||
"google shopping":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
|
||||
"google books":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
|
||||
"youtube":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
|
||||
"google videos":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
|
||||
"google images":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
|
||||
"yahoo":
|
||||
"https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
|
||||
"bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
|
||||
"ebay":
|
||||
"http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
|
||||
"amazon":
|
||||
"http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
|
||||
"twitter":
|
||||
"https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
|
||||
"baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
|
||||
"yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
|
||||
"google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
|
||||
"google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
// "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
|
||||
},
|
||||
/**
|
||||
* Get the service url based on options set in the dom.
|
||||
* @return {String} A jsonp url for search suggestions with query missing from the end.
|
||||
*/
|
||||
getUrl :function(){
|
||||
// Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
|
||||
// Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
|
||||
// Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
|
||||
// https://developers.google.com/custom-search/docs/ref_languages
|
||||
return _.template(this.services[this.options.service])(this.options);
|
||||
},
|
||||
|
||||
|
||||
/** Parse response per service **/
|
||||
parseServiceResponse: function(res){
|
||||
// Each take a json response tand return a keyword array
|
||||
RESPONSE_TEMPLATES = {
|
||||
// opensearch default
|
||||
"default": function (res) {
|
||||
return res[1];
|
||||
},
|
||||
"yahoo": function (res) {
|
||||
return _.map(res.gossip.results, 'key');
|
||||
},
|
||||
"ebay": function (res) {
|
||||
return res.res ? res.res.sug : [];
|
||||
},
|
||||
"twitter": function (res) {
|
||||
return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
|
||||
},
|
||||
"baidu": function (res) {
|
||||
return res.s;
|
||||
},
|
||||
"yandex": function(res){
|
||||
return _.map(res[1], function(r){
|
||||
return typeof r === 'string' ? r : r[1];
|
||||
});
|
||||
},
|
||||
"linkedin": function(res){
|
||||
return _.map(res.resultList,'displayName');
|
||||
},
|
||||
"google play": function(res){return _.map(res,'s')},
|
||||
"google play apps": function(res){return _.map(res,'s')},
|
||||
"google play movies": function(res){return _.map(res,'s')},
|
||||
"google play books": function(res){return _.map(res,'s')},
|
||||
};
|
||||
var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"];
|
||||
return parser(res);
|
||||
},
|
||||
|
||||
setUpDb: function(success){
|
||||
var self=this;
|
||||
// setup a db. Ref: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB
|
||||
var dbReq = window.indexedDB.open("KeywordShitter2", 3);
|
||||
/** Error handler for all child transactions as events bubbe up **/
|
||||
dbReq.onerror = function (event) {
|
||||
return console.error('Error opening indexedDB database KeywordShitter2', event);
|
||||
};
|
||||
dbReq.onsuccess = function (event) {
|
||||
self.db = event.target.result;
|
||||
self.db.onerror = function (event) {
|
||||
// Generic error handler requests
|
||||
if (event && event.target && event.target.error)
|
||||
console.error("Database error",event.target.error.errorCode, event.target.error.message, event);
|
||||
else
|
||||
console.error("Database error",arguments);
|
||||
return this;
|
||||
};
|
||||
if (success) success(self.db,dbReq);
|
||||
return self.db;
|
||||
};
|
||||
dbReq.onupgradeneeded = function (event) {
|
||||
console.log("running onupgradeneeded");
|
||||
self.db = event.target.result;
|
||||
|
||||
if (!self.db.objectStoreNames.contains("suggestions")) {
|
||||
var objectStore = self.db.createObjectStore("suggestions", {
|
||||
autoIncrement: true, keyPath: 'id'
|
||||
});
|
||||
// Create an index to search suggestions by
|
||||
// he query that prompted the suggestion
|
||||
if (!objectStore.indexNames.contains('search'))
|
||||
objectStore.createIndex("search", "search", {unique: false});
|
||||
|
||||
// and by suggestion/keyword
|
||||
if (!objectStore.indexNames.contains('keyword'))
|
||||
objectStore.createIndex("keyword", "keyword", {unique: false});
|
||||
|
||||
}
|
||||
|
||||
return self.db;
|
||||
};
|
||||
return dbReq;
|
||||
},
|
||||
services:suggestions.services,
|
||||
getUrl :suggestions.getUrl,
|
||||
parseServiceResponse: suggestions.parseServiceResponse,
|
||||
|
||||
toggleWork: function(){
|
||||
if (this.doWork === false)
|
||||
@@ -165,7 +42,7 @@ var KWS = function(){
|
||||
|
||||
// get queries from the input
|
||||
var ks = $('#input').val().split("\n");
|
||||
this.keywordsToQuery=ks.map(this.CleanVal);
|
||||
this.keywordsToQuery=_.map(ks,this.CleanVal);
|
||||
|
||||
|
||||
// add variations of the initial terms
|
||||
@@ -238,7 +115,7 @@ var KWS = function(){
|
||||
},
|
||||
|
||||
addResultsToQueue: function(retList, search){
|
||||
retList=retList.map(this.CleanVal);
|
||||
retList=_.map(retList,this.CleanVal);
|
||||
|
||||
// add each result to list first before permutations
|
||||
for (var j = 0; j < retList.length; j++) {
|
||||
@@ -255,6 +132,7 @@ var KWS = function(){
|
||||
permuteResultsToQueue: function(retList, search){
|
||||
var chr, currentx, currentKw;
|
||||
var self = this;
|
||||
var options = this.getOptions()
|
||||
|
||||
this.hashMapInputs[search] = true;
|
||||
|
||||
@@ -270,14 +148,14 @@ var KWS = function(){
|
||||
return s+' '+suffix;
|
||||
}
|
||||
// clean
|
||||
retList=retList.map(this.CleanVal);
|
||||
retList=_.map(retList,this.CleanVal);
|
||||
|
||||
// get permutations
|
||||
var newInputs = retList.reduce(function(result, keyword){
|
||||
return _.concat(
|
||||
result,
|
||||
self.options.prefixes.map(addPrefix.bind(self,keyword)),
|
||||
self.options.suffixes.map(addSuffix.bind(self,keyword))
|
||||
_.map(options.prefixes,addPrefix.bind(self,keyword)),
|
||||
_.map(options.suffixes,addSuffix.bind(self,keyword))
|
||||
);
|
||||
}, []);
|
||||
|
||||
@@ -287,66 +165,12 @@ var KWS = function(){
|
||||
return newInputs;
|
||||
},
|
||||
|
||||
/** export db as json **/
|
||||
exportDB: function(success){
|
||||
var reqObj = this.db.transaction(["suggestions"],"readonly")
|
||||
.objectStore("suggestions")
|
||||
.getAll();
|
||||
reqObj.onsuccess = function(e) {
|
||||
var blob, name;
|
||||
if (e.target.result.length){
|
||||
var jsonData = JSON.stringify(e.target.result);
|
||||
blob = new Blob([jsonData], {type: "octet/stream"});
|
||||
var timeStamp = (new Date()).toISOString().replace(/[:\.]/g,'_').slice(0,-5);
|
||||
name = 'keywordshitter_'+timeStamp+'_r'+e.target.result.length+'.json';
|
||||
saveAs(blob,name);
|
||||
if (success instanceof Function) success(blob);
|
||||
}
|
||||
return blob;
|
||||
};
|
||||
return reqObj;
|
||||
},
|
||||
|
||||
clearDB: function(){
|
||||
this.db.transaction(["suggestions"], "readwrite")
|
||||
.objectStore("suggestions")
|
||||
.clear();
|
||||
console.warn('cleared all indexedDB data');
|
||||
},
|
||||
|
||||
/** Display data from db upon pressing load button **/
|
||||
loadFromDB: function(){
|
||||
var self = this;
|
||||
var reqObj = this.db.transaction(["suggestions"],"readonly")
|
||||
.objectStore("suggestions")
|
||||
.getAll()
|
||||
.onsuccess = function(e) {
|
||||
if (e.target.result.length){
|
||||
/// grab the fields we want
|
||||
var data =[];
|
||||
for (var i = 0; i < e.target.result.length; i++) {
|
||||
var da = e.target.result[i];
|
||||
|
||||
// also remove undefined so datatables doesn't bring up alerts
|
||||
da = _.mapValues(da, function(v){return v===undefined ? null: v;});
|
||||
|
||||
// parse nums
|
||||
// da = da.map(v => /^[\d+\.]+$/.test(v) ? Number(v): v);
|
||||
//
|
||||
data.push(da);
|
||||
}
|
||||
self.table.rows.add(data);
|
||||
return self.table.draw(false);
|
||||
}
|
||||
};
|
||||
return;
|
||||
},
|
||||
|
||||
/** Display results **/
|
||||
displayResults: function(retList, search, dontDisplay, url,data){
|
||||
|
||||
var rows=[];
|
||||
retList=retList.map(this.CleanVal);
|
||||
retList=_.map(retList,this.CleanVal);
|
||||
for (var i = 0; i < retList.length; i++) {
|
||||
var cleanKw = retList[i];
|
||||
|
||||
@@ -411,49 +235,6 @@ var KWS = function(){
|
||||
return domain;
|
||||
},
|
||||
|
||||
/** Store new results in db and hashmap **/
|
||||
storeResults: function(retList, search, url){
|
||||
var self = this;
|
||||
retList=retList.map(this.CleanVal);
|
||||
|
||||
// We will add the items async in order
|
||||
// Ref: http://stackoverflow.com/a/13666741/221742
|
||||
var transaction = this.db.transaction(["suggestions"], "readwrite");
|
||||
var store = transaction.objectStore("suggestions");
|
||||
var i=0;
|
||||
addNext();
|
||||
|
||||
/** Like an async for loop to add each to db **/
|
||||
function addNext() {
|
||||
if (i<retList.length) {
|
||||
var cleanKw = retList[i];
|
||||
if (cleanKw.length){
|
||||
addReq = store.add({
|
||||
keyword: cleanKw,
|
||||
length: cleanKw.length,
|
||||
words: cleanKw.trim().split(/ +/).length,
|
||||
search: search,
|
||||
ip: this.myIp,
|
||||
url: url,
|
||||
domain: self.extractDomain(url),
|
||||
time: (new Date()).toUTCString(),
|
||||
});
|
||||
addReq.onsuccess = addNext;
|
||||
|
||||
++i;
|
||||
return addReq;
|
||||
} else {
|
||||
// skip empty keywords
|
||||
++i;
|
||||
return addNext.bind(this)();
|
||||
}
|
||||
} else {
|
||||
return;// console.debug('populate complete for suggestions of ', search);
|
||||
}
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
/** mark a search as done in the queue **/
|
||||
markAsDone: function(search){
|
||||
// mark as done in queue
|
||||
@@ -481,61 +262,38 @@ var KWS = function(){
|
||||
var self = this;
|
||||
this.queryLock = true;
|
||||
|
||||
// first check in db
|
||||
var reqObj = this.db.transaction(["suggestions"],"readonly")
|
||||
.objectStore("suggestions")
|
||||
.index("search")
|
||||
.getAll(search);
|
||||
reqObj.onsuccess = function(e) {
|
||||
var domain = self.extractDomain(self.getUrl());
|
||||
var results = e.target.result.filter(function(r){return r.domain==domain;});
|
||||
// console.log(e.target.result);
|
||||
if (results.length){
|
||||
// search was done previously so display results from db
|
||||
var retList=_.map(results,'keyword');
|
||||
self.markAsDone(search);
|
||||
self.displayResults(retList,search,undefined,undefined,results);
|
||||
// search not done, lets do the query
|
||||
url = self.getUrl()+search;
|
||||
var promise = $.ajax({
|
||||
url: url,
|
||||
// jsonp: "jsonp",
|
||||
dataType: "jsonp",
|
||||
success: function (res, statusText, jqXHR) {
|
||||
var retList = self.parseServiceResponse(res);
|
||||
if (retList && retList.length){
|
||||
// self.storeResults(retList, search, this.url);
|
||||
self.displayResults(retList, search, undefined, this.url);
|
||||
self.addResultsToQueue(retList);
|
||||
if (self.options.keepRunning) self.permuteResultsToQueue(retList);
|
||||
self.queryLock = false;
|
||||
self.markAsDone(search);
|
||||
} else {
|
||||
// console.debug('No suggestions for query: "',search,'"');
|
||||
self.markAsNone(search);
|
||||
}
|
||||
else {
|
||||
// search not done, lets do the query
|
||||
url = self.getUrl()+search;
|
||||
$.ajax({
|
||||
url: url,
|
||||
// jsonp: "jsonp",
|
||||
dataType: "jsonp",
|
||||
success: function (res, statusText, jqXHR) {
|
||||
self.queryLock = false;
|
||||
return;
|
||||
|
||||
var retList = self.parseServiceResponse(res);
|
||||
var char, currentx;
|
||||
if (retList && retList.length){
|
||||
self.storeResults(retList, search, this.url);
|
||||
self.displayResults(retList, search, undefined, this.url);
|
||||
self.addResultsToQueue(retList);
|
||||
if (self.options.keepRunning) self.permuteResultsToQueue(retList);
|
||||
self.markAsDone(search);
|
||||
} else {
|
||||
// console.debug('No suggestions for query: "',search,'"');
|
||||
self.markAsNone(search);
|
||||
}
|
||||
self.queryLock = false;
|
||||
return;
|
||||
|
||||
},
|
||||
error: function(jqXHR,errorText,error){
|
||||
console.error(errorText,this.url,this,jqXHR,error);
|
||||
self.queryLock = false;
|
||||
return;
|
||||
},
|
||||
callback: function(){
|
||||
console.log(this,arguments);
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
return reqObj;
|
||||
},
|
||||
error: function(jqXHR,errorText,error){
|
||||
console.error(errorText,this.url,this,jqXHR,error);
|
||||
self.queryLock = false;
|
||||
return;
|
||||
},
|
||||
callback: function(){
|
||||
console.log(this,arguments);
|
||||
}
|
||||
});
|
||||
return promise;
|
||||
},
|
||||
|
||||
/** Clean input, may not all be needed **/
|
||||
@@ -685,7 +443,7 @@ var KWS = function(){
|
||||
|
||||
/** load settings from localStorage **/
|
||||
loadSettings: function(){
|
||||
// TODO do table settings as well, e.g. column visibilitity
|
||||
// Tabe settings are auto handles by datatables
|
||||
if (localStorage.service) $("#service").val( localStorage.service );
|
||||
if (localStorage.country) $('#country').val(localStorage.country);
|
||||
if (localStorage.lang) $('#lang').val(localStorage.lang);
|
||||
@@ -721,7 +479,7 @@ var KWS = function(){
|
||||
},
|
||||
|
||||
init: function(){
|
||||
this.setUpDb();
|
||||
// this.setUpDb();
|
||||
|
||||
// add this.servicess to search engine settings
|
||||
for (var service in this.services) {
|
||||
@@ -739,15 +497,11 @@ var KWS = function(){
|
||||
$('#progress1').addClass('progressjs-progress');
|
||||
this.progress1 = progressJs("#progress1");
|
||||
|
||||
|
||||
// bind buttons
|
||||
$('#startjob').on('click',this.toggleWork.bind(this));
|
||||
$('#reset').on('click',this.reset.bind(this));
|
||||
$('#load-from-cache').on('click',this.loadFromDB.bind(this));
|
||||
$('#export-from-cache').on('click',this.exportDB.bind(this));
|
||||
$('#clear-cache').on('click',this.clearDB.bind(this));
|
||||
|
||||
|
||||
|
||||
// setup table
|
||||
this.table = $('#outtable').DataTable({
|
||||
pageLength: 25,
|
||||
"lengthMenu": [ 10, 25, 50, 75, 100,800],
|
||||
@@ -768,16 +522,32 @@ var KWS = function(){
|
||||
extend: 'csvHtml5',
|
||||
fieldBoundary: "",
|
||||
text: 'Copy keywords',
|
||||
// 'customize': function(data,options){return data.split('\n').join(',');},
|
||||
// 'customize': function(data,options){
|
||||
// console.log(data,options);return data.split('\n').join(',');
|
||||
// },
|
||||
header: false,
|
||||
exportOptions: {
|
||||
stripNewlines: true,
|
||||
stripHtml: true,
|
||||
decodeEntities: true,
|
||||
columns: 'keyword',
|
||||
// format:{
|
||||
// body: function(html,i){console.log(html);return html}
|
||||
// }
|
||||
columns: 1,
|
||||
// format:{
|
||||
// body: function(html,i){
|
||||
// console.log(html);return html
|
||||
// }
|
||||
// }
|
||||
}
|
||||
},
|
||||
{
|
||||
extend: 'csvHtml5',
|
||||
fieldBoundary: "",
|
||||
text: 'Copy visible columns',
|
||||
header: false,
|
||||
exportOptions: {
|
||||
columns: ':visible',
|
||||
stripNewlines: true,
|
||||
stripHtml: true,
|
||||
decodeEntities: true,
|
||||
}
|
||||
},
|
||||
]
|
||||
@@ -791,6 +561,7 @@ var KWS = function(){
|
||||
"targets": 0,
|
||||
"visible": false,
|
||||
}, {
|
||||
"name": "keyword",
|
||||
"title": "Keyword",
|
||||
"data": "keyword",
|
||||
"responsivePriority": 1,
|
||||
@@ -849,6 +620,7 @@ var KWS = function(){
|
||||
// scroller: true
|
||||
});
|
||||
|
||||
// get user ip
|
||||
$.getJSON('https://api.ipify.org?format=json', function (data) {
|
||||
this.myIp = data.ip;
|
||||
});
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"name": "keywordshitter2",
|
||||
"version": "0.0.5",
|
||||
"description": "The second generation of keyword shitter. Bigger, better, and more pungent than other products.",
|
||||
"main": "server.js",
|
||||
"main": "public/js/main.js",
|
||||
"repository": "https://github.com/wassname/keywordshitter2",
|
||||
"scripts": {
|
||||
"test": "public/test.html",
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
var suggestions = function(){
|
||||
return {
|
||||
defaultOptions:{
|
||||
deferTableUpdatesAtRows: 5000,
|
||||
keepRunning: false,
|
||||
maxQueueDisplay: 5000,
|
||||
country: "",
|
||||
filterNegative: "",
|
||||
filterPositive: "",
|
||||
lang: "",
|
||||
prefixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "how", "which", "why", "where", "who", "when", "are", "what"],
|
||||
rateLimit: 750,
|
||||
service: "google",
|
||||
suffixes: [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "y", "x", "y", "z", "like", "for", "without", "with", "versus", "vs", "to", "near", "except", "has"]
|
||||
},
|
||||
|
||||
services:{
|
||||
"google":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
|
||||
"google news":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
|
||||
"google shopping":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
|
||||
"google books":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
|
||||
"youtube":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
|
||||
"google videos":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
|
||||
"google images":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
|
||||
"yahoo":
|
||||
"https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
|
||||
"bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
|
||||
"ebay":
|
||||
"http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
|
||||
"amazon":
|
||||
"http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
|
||||
"twitter":
|
||||
"https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
|
||||
"baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
|
||||
"yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
|
||||
"google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
|
||||
"google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
// "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
|
||||
},
|
||||
/**
|
||||
* Get the service url based on options set in the dom.
|
||||
* @return {String} A jsonp url for search suggestions with query missing from the end.
|
||||
*/
|
||||
getUrl :function(service, options){
|
||||
// Ref: https://github.com/estivo/Instantfox/blob/master/firefox/c1hrome/content/defaultPluginList.js
|
||||
// Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
|
||||
// Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
|
||||
// https://developers.google.com/custom-search/docs/ref_languages
|
||||
options = _.defaults({},this.defaultOptions, this.getOptions(), options)
|
||||
return _.template(this.services[(service||this.options.service)])(options);
|
||||
},
|
||||
|
||||
|
||||
/** Parse response per service **/
|
||||
parseServiceResponse: function(res, service){
|
||||
// Each take a json response tand return a keyword array
|
||||
RESPONSE_TEMPLATES = {
|
||||
// opensearch default
|
||||
"default": function (res) {
|
||||
return res[1];
|
||||
},
|
||||
"yahoo": function (res) {
|
||||
return _.map(res.gossip.results, 'key');
|
||||
},
|
||||
"ebay": function (res) {
|
||||
return res.res ? res.res.sug : [];
|
||||
},
|
||||
"twitter": function (res) {
|
||||
return _.concat(res.users, _.map(res.topics, 'topic'), res.hashtags, res.oneclick);
|
||||
},
|
||||
"baidu": function (res) {
|
||||
return res.s;
|
||||
},
|
||||
"yandex": function(res){
|
||||
return _.map(res[1], function(r){
|
||||
return typeof r === 'string' ? r : r[1];
|
||||
});
|
||||
},
|
||||
"linkedin": function(res){
|
||||
return _.map(res.resultList,'displayName');
|
||||
},
|
||||
"google play": function(res){return _.map(res,'s')},
|
||||
"google play apps": function(res){return _.map(res,'s')},
|
||||
"google play movies": function(res){return _.map(res,'s')},
|
||||
"google play books": function(res){return _.map(res,'s')},
|
||||
};
|
||||
var parser = RESPONSE_TEMPLATES[(service||this.options.service)] || RESPONSE_TEMPLATES["default"];
|
||||
return parser(res);
|
||||
}
|
||||
}
|
||||
}()
|
||||
+10
-33
@@ -16,35 +16,15 @@ describe('permute', function(){
|
||||
it('should correctly permute results and add to queue', function() {
|
||||
var retList = ['a','b','c d',' a longer one'];
|
||||
KWS.permuteResultsToQueue(retList,'test');
|
||||
assert(KWS.keywordsToQuery.length>0);
|
||||
assert(KWS.hashMapInputs['b']);
|
||||
assert(KWS.hashMapInputs['test']);
|
||||
expect(KWS.keywordsToQuery).to.have.length.above(0)
|
||||
// assert(KWS.hashMapInputs['b']);
|
||||
expect(KWS.hashMapInputs).to.include.key('test');
|
||||
// var options = KWS.getOptions();
|
||||
// var expectedLength = options.prefixes.length*retList.length+options.suffixes.length*retList.length
|
||||
// assert(KWS.keywordsToQuery.length==expectedLength,''+expectedLength+'!='+KWS.keywordsToQuery.length);
|
||||
});
|
||||
});
|
||||
|
||||
// TODO use localForage or most popular promise wrapper for indexedDB then test
|
||||
// describe('exportDB', function(){
|
||||
// });
|
||||
|
||||
describe('db', function() {
|
||||
it('should correctly set up db', function(done) {
|
||||
var dbReq=KWS.setUpDb();
|
||||
var oldonsuccess=dbReq.onsuccess;
|
||||
dbReq.onsuccess=function(event){
|
||||
// override onld onsucess because it doesn't use promises
|
||||
var db=oldonsuccess.bind(dbReq)(event);
|
||||
assert(db);
|
||||
assert(db.objectStoreNames.contains('suggestions'),'db should contain suggestons store');
|
||||
var indexNames =db.transaction('suggestions','readonly').objectStore('suggestions').indexNames;
|
||||
assert(indexNames.contains('keyword','search'));
|
||||
done();
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('extractDomain', function(){
|
||||
|
||||
@@ -66,11 +46,10 @@ describe('responses', function(){
|
||||
if (testData.responses.hasOwnProperty(service)) {
|
||||
it('should correctly parse test data for service='+service, function() {
|
||||
var res = testData.responses[service];
|
||||
var options = {service:service};
|
||||
var parsedRes = KWS.parseServiceResponse(res,options);
|
||||
var parsedRes = KWS.parseServiceResponse(res,service);
|
||||
|
||||
assert.typeOf(parsedRes,'array');
|
||||
expect(parsedRes).to.have.length.above(1);
|
||||
expect(parsedRes).to.be.an('array')
|
||||
.that.has.length.above(1);
|
||||
parsedRes.map(r => assert.typeOf(r, 'string'));
|
||||
});
|
||||
}
|
||||
@@ -101,8 +80,7 @@ describe('services', function() {
|
||||
services.forEach(function(service){
|
||||
searches.forEach(function(search){
|
||||
it('should correctly get and parse data'+'. Service="'+service+'" search="'+search+'"', function(done) {
|
||||
var options = {service:service};
|
||||
var url = KWS.getUrl(options)+search;
|
||||
var url = KWS.getUrl(service)+search;
|
||||
return $.ajax({
|
||||
url: url,
|
||||
async: false,
|
||||
@@ -111,7 +89,7 @@ describe('services', function() {
|
||||
success: function (res, statusText, jqXHR) {
|
||||
assert(statusText=="success");
|
||||
assert(res!==[]);
|
||||
var parsedRes = KWS.parseServiceResponse(res,options);
|
||||
var parsedRes = KWS.parseServiceResponse(res,service);
|
||||
assert.typeOf(parsedRes,'array');
|
||||
parsedRes.map(r => assert.typeOf(r, 'string'));
|
||||
done();
|
||||
@@ -129,8 +107,7 @@ describe('services', function() {
|
||||
services.forEach(function(service){
|
||||
searchesDifficult.forEach(function(search){
|
||||
it('should get and parse difficult data'+'. Service="'+service+'" search="'+search+'"', function(done) {
|
||||
var options = {service:service};
|
||||
var url = KWS.getUrl(options)+search;
|
||||
var url = KWS.getUrl(service)+search;
|
||||
return $.ajax({
|
||||
url: url,
|
||||
async: false,
|
||||
@@ -140,7 +117,7 @@ describe('services', function() {
|
||||
assert(statusText=="success");
|
||||
assert(res!==[]);
|
||||
|
||||
var parsedRes = KWS.parseServiceResponse(res,options);
|
||||
var parsedRes = KWS.parseServiceResponse(res,service);
|
||||
assert.typeOf(parsedRes,'array');
|
||||
parsedRes.map(r => assert.typeOf(r, 'string'));
|
||||
return done();
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/localforage/1.3.3/localforage.js"></script>
|
||||
|
||||
<script src="js/suggestions.js"></script>
|
||||
<script src="js/main.js"></script>
|
||||
<script src="js/testData.js"></script>
|
||||
<script src="js/test.js"></script>
|
||||
|
||||
Reference in New Issue
Block a user