mirror of
https://github.com/wassname/keywordshitter2.git
synced 2026-06-27 16:10:23 +08:00
Autogen service options
This commit is contained in:
+7
-2
@@ -8,15 +8,20 @@
|
||||
"test": "public/test.html",
|
||||
"build-css": "lessc ./src/less/main.less > public/css/style.css",
|
||||
"build": "npm run build-css",
|
||||
"start": "node server.js"
|
||||
"start": "node server.js",
|
||||
"debug": "NODE_ENV=dev DEBUG=* nodemon server",
|
||||
"production": "NODE_ENV=production npm start"
|
||||
},
|
||||
"author": "blackmia,wassname",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"chai": "^3.5.0",
|
||||
"cors": "^2.7.1",
|
||||
"debug": "^2.2.0",
|
||||
"express": "^4.13.4",
|
||||
"lessc": "latest",
|
||||
"mocha": "^2.4.5"
|
||||
"mocha": "^2.4.5",
|
||||
"morgan": "^1.6.1",
|
||||
"winston": "^2.1.1"
|
||||
}
|
||||
}
|
||||
|
||||
+38
-31
@@ -72,7 +72,7 @@ lying internet</textarea>
|
||||
<div class="form-group">
|
||||
<label for="service">Service:</label>
|
||||
<select class="form-control" name="service" id="service" >
|
||||
<option>google</option>
|
||||
<!-- <option>google</option>
|
||||
<option>twitter</option>
|
||||
<option>yahoo</option>
|
||||
<option>bing</option>
|
||||
@@ -86,8 +86,32 @@ lying internet</textarea>
|
||||
<option>google shopping</option>
|
||||
<option>yandex</option>
|
||||
<option>baidu</option>
|
||||
<option>linkedin</option> -->
|
||||
<!-- <option>yelp</option> -->
|
||||
</select>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="form-group">
|
||||
<label for="prefixes">Prefixes</label>
|
||||
<textarea type="text" title="These values are added before words to prompt the search suggestions" class="form-control" name="prefixes" id="prefixes" > ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,how,which,why,where,who,when,are,what</textarea>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="suffixes">Suffixes:</label>
|
||||
<textarea type="text" rows="2" title="These values are added after words to prompt the search suggestions" class="form-control" id="suffixes"> ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,like,for,without,with,versus,vs,to,near,except,has</textarea>
|
||||
</div>
|
||||
|
||||
<div class="form-group" style="display: none">
|
||||
<label for="rate-limit">Rate limit (for experts):</label>
|
||||
<input type="text" class="form-control" id="rate-limit" value="750" name="rate-limit" title="Value between searches in milliseconds only change if you know what you are doing. This could get your ip banned, or place an unfair burden on the suggest servers." disabled>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label for="keep-running">
|
||||
<input type="checkbox" id="keep-running" name="keep-running" title="Keep running forever"></input>Continuous running:
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="country">Country code (google):</label>
|
||||
<!-- ISO-3366-1: Alpha-2 Codes -->
|
||||
@@ -389,34 +413,6 @@ lying internet</textarea>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="prefixes">Prefixes</label>
|
||||
<textarea type="text" title="These values are added before words to prompt the search suggestions" class="form-control" name="prefixes" id="prefixes" > ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,how,which,why,where,who,when,are,what</textarea>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="suffixes">Suffixes:</label>
|
||||
<textarea type="text" rows="2" title="These values are added after words to prompt the search suggestions" class="form-control" id="suffixes"> ,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,y,x,y,z,like,for,without,with,versus,vs,to,near,except,has</textarea>
|
||||
</div>
|
||||
|
||||
<div class="form-group" style="display: none">
|
||||
<label for="filter-positive" >Positive filter:</label>
|
||||
<textarea disabled id="filter-positive" name="filter-positive" class="form-control" rows="2"
|
||||
placeholder="Positive Filter"></textarea>
|
||||
</div>
|
||||
<div class="form-group" style="display: none">
|
||||
<label for="filter-negative">Negative filter:</label>
|
||||
<textarea disabled id="filter-negative" name="filter-negative" class="form-control" rows="2"placeholder="Negative Filter"></textarea>
|
||||
</div>
|
||||
<div class="form-group" style="display: none">
|
||||
<label for="rate-limit">Rate limit (for experts):</label>
|
||||
<input type="text" class="form-control" id="rate-limit" value="750" name="rate-limit" title="Value between searches in milliseconds only change if you know what you are doing. This could get your ip banned, or place an unfair burden on the suggest servers." disabled>
|
||||
</div>
|
||||
|
||||
<div class="checkbox">
|
||||
<label for="keep-running">
|
||||
<input type="checkbox" id="keep-running" name="keep-running" title="Keep running forever"></input>Continuous running:
|
||||
</label>
|
||||
</div>
|
||||
|
||||
</form>
|
||||
|
||||
@@ -471,11 +467,11 @@ lying internet</textarea>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/datatables/1.10.10/js/jquery.js"></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.2.0/jquery.js"></script>
|
||||
<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" crossorigin="anonymous"></script>
|
||||
<script src="//cdn.datatables.net/s/bs/jszip-2.5.0,dt-1.10.10,b-1.1.0,b-colvis-1.1.0,b-html5-1.1.0,b-print-1.1.0,cr-1.3.0,fh-3.1.0,kt-2.1.0,r-2.0.0,se-1.1.0/datatables.min.js" crossorigin="anonymous"></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/4.2.0/lodash.min.js"></script>
|
||||
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
|
||||
<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
|
||||
<!-- <script src="//cdnjs.cloudflare.com/ajax/libs/localforage/1.3.3/localforage.js"></script> -->
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/progress.js/0.1.0/progress.min.js"></script>
|
||||
|
||||
@@ -488,6 +484,17 @@ lying internet</textarea>
|
||||
<a href="https://github.com/wassname/keywordshitter2"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://camo.githubusercontent.com/652c5b9acfaddf3a9c326fa6bde407b87f7be0f4/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f6f72616e67655f6666373630302e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"></a>
|
||||
<!-- Fork me on github... Do you get it? Do you! -->
|
||||
|
||||
<!-- Lets see if anyone ever uses this -->
|
||||
<script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-51809277-6', 'auto');
|
||||
ga('send', 'pageview');
|
||||
|
||||
</script>
|
||||
<script>
|
||||
$(document).ready(function () {
|
||||
KWS.init();
|
||||
|
||||
+69
-48
@@ -13,6 +13,38 @@ var KWS = function(){
|
||||
keywordsToQuery: [],
|
||||
keywordsToQueryIndex: 0,
|
||||
numOfInitialKeywords: 0,
|
||||
services:{
|
||||
"google":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
|
||||
"google news":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
|
||||
"google shopping":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
|
||||
"google books":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
|
||||
"youtube":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
|
||||
"google videos":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
|
||||
"google images":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
|
||||
"yahoo":
|
||||
"https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
|
||||
"bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
|
||||
"ebay":
|
||||
"http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
|
||||
"amazon":
|
||||
"http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
|
||||
"twitter":
|
||||
"https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
|
||||
"baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
|
||||
"yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
|
||||
"google play": "https://market.android.com/suggest/SuggRequest?json=1&c=0&hl=${lang}&gl=${country}&callback=?&query=", //
|
||||
"google play apps": "https://market.android.com/suggest/SuggRequest?json=1&c=3&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play movies": "https://market.android.com/suggest/SuggRequest?json=1&c=4&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
"google play books": "https://market.android.com/suggest/SuggRequest?json=1&c=1&hl=${lang}&gl=${country}&callback=?&query=",
|
||||
// "kickasstorrents": "https://kat.cr/get_queries.php?query=", // not jsonp
|
||||
},
|
||||
/**
|
||||
* Get the service url based on options set in the dom.
|
||||
* @return {String} A jsonp url for search suggestions with query missing from the end.
|
||||
@@ -22,35 +54,7 @@ var KWS = function(){
|
||||
// Ref: https://github.com/bnoordhuis/mozilla-central/tree/master/browser/locales/en-US/searchplugins
|
||||
// Ref: https://developers.google.com/custom-search/json-api/v1/reference/cse/list
|
||||
// https://developers.google.com/custom-search/docs/ref_languages
|
||||
services={
|
||||
"google":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&gl=${country}&callback=?&q=",
|
||||
"google news":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=n&gl=${country}&callback=?&q=",
|
||||
"google shopping":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=sh&gl=${country}&callback=?&q=",
|
||||
"google books":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=bo&gl=${country}&callback=?&q=",
|
||||
"youtube":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=yt&gl=${country}&callback=?&q=",
|
||||
"google videos":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=v&gl=${country}&callback=?&q=",
|
||||
"google images":
|
||||
"http://suggestqueries.google.com/complete/search?client=chrome&hl=${lang}&ds=i&gl=${country}&callback=?&q=",
|
||||
"yahoo":
|
||||
"https://search.yahoo.com/sugg/ff?output=jsonp&appid=ffd&callback=?&command=",
|
||||
"bing": "http://api.bing.com/osjson.aspx?JsonType=callback&JsonCallback=?&query=",
|
||||
"ebay":
|
||||
"http://autosug.ebay.com/autosug?_jgr=1&sId=0&_ch=0&callback=?&kwd=",
|
||||
"amazon":
|
||||
"http://completion.amazon.co.uk/search/complete?method=completion&search-alias=aps&mkt=4&callback=?&q=",
|
||||
"twitter":
|
||||
"https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=?&q=",
|
||||
"baidu": "http://suggestion.baidu.com/su?cb=?&wd=",
|
||||
"yandex": "https://yandex.com/suggest/suggest-ya.cgi?callback=?&q=?&n=30&v=4&uil={lang}&part=",
|
||||
};
|
||||
|
||||
return _.template(services[this.options.service])(this.options);
|
||||
return _.template(this.services[this.options.service])(this.options);
|
||||
},
|
||||
|
||||
|
||||
@@ -78,7 +82,14 @@ var KWS = function(){
|
||||
return _.map(res[1], function(r){
|
||||
return typeof r === 'string' ? r : r[1];
|
||||
});
|
||||
}
|
||||
},
|
||||
"linkedin": function(res){
|
||||
return _.map(res.resultList,'displayName');
|
||||
},
|
||||
"google play": function(res){return _.map(res,'s')},
|
||||
"google play apps": function(res){return _.map(res,'s')},
|
||||
"google play movies": function(res){return _.map(res,'s')},
|
||||
"google play books": function(res){return _.map(res,'s')},
|
||||
};
|
||||
var parser = RESPONSE_TEMPLATES[this.options.service] || RESPONSE_TEMPLATES["default"];
|
||||
return parser(res);
|
||||
@@ -493,7 +504,7 @@ var KWS = function(){
|
||||
url = self.getUrl()+search;
|
||||
$.ajax({
|
||||
url: url,
|
||||
jsonp: "jsonp",
|
||||
// jsonp: "jsonp",
|
||||
dataType: "jsonp",
|
||||
success: function (res, statusText, jqXHR) {
|
||||
|
||||
@@ -513,10 +524,13 @@ var KWS = function(){
|
||||
return;
|
||||
|
||||
},
|
||||
error: function(e){
|
||||
console.error(e);
|
||||
error: function(jqXHR,errorText,error){
|
||||
console.error(errorText,this.url,this,jqXHR,error);
|
||||
self.queryLock = false;
|
||||
return;
|
||||
},
|
||||
callback: function(){
|
||||
console.log(this,arguments);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -526,26 +540,23 @@ var KWS = function(){
|
||||
|
||||
/** Clean input, may not all be needed **/
|
||||
CleanVal: function(input) {
|
||||
// We want to clean search terms but it's not possible to do this perfectly
|
||||
// as differen't search engines strip differen't amounts from the term
|
||||
// so we will keep as much details as possible
|
||||
|
||||
function escapeHtml(unsafe) {
|
||||
return unsafe
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
// Search engines are sensitive to whitespace so we do not want to trim
|
||||
// Some return html or escaped html, so we do want to convert to text
|
||||
|
||||
// google returns lowercase
|
||||
// removed escaped html and html tags
|
||||
// e.g. '<b>A&M</b>' => 'A&M'
|
||||
input=$('<div />').html(input).text();
|
||||
|
||||
// I don't know of any search engines sentitive or case so make all lowercase
|
||||
input = input.toLowerCase();
|
||||
input = escapeHtml(input);
|
||||
|
||||
// not sure if we want to trim, the search engines are sensitive to
|
||||
// whitespace. (and also last search)
|
||||
// input = input.trim();
|
||||
|
||||
// this removes navigation suggestions
|
||||
// this removes navigation suggestions, perhaps we need to move this to result parser
|
||||
if (input.length > 4 && input.substring(0, 4) == "http") input = "";
|
||||
|
||||
return input;
|
||||
},
|
||||
|
||||
@@ -711,6 +722,15 @@ var KWS = function(){
|
||||
|
||||
init: function(){
|
||||
this.setUpDb();
|
||||
|
||||
// add this.servicess to search engine settings
|
||||
for (var service in this.services) {
|
||||
if (this.services.hasOwnProperty(service)) {
|
||||
$('#service').append('<option>'+service+'</option>')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
this.loadSettings();
|
||||
this.options = this.getOptions();
|
||||
|
||||
@@ -727,6 +747,7 @@ var KWS = function(){
|
||||
$('#clear-cache').on('click',this.clearDB.bind(this));
|
||||
|
||||
|
||||
|
||||
this.table = $('#outtable').DataTable({
|
||||
pageLength: 25,
|
||||
"lengthMenu": [ 10, 25, 50, 75, 100,800],
|
||||
|
||||
+2
-1
@@ -84,7 +84,8 @@ describe('services', function() {
|
||||
this.slow(2000);
|
||||
|
||||
// data
|
||||
var services = ["google", "google news", "google shopping", "google books", "youtube", "google videos", "google images", "yahoo", "bing", "ebay", "amazon", "twitter", "baidu", "yandex"];
|
||||
// var services = ["google", "google news", "google shopping", "google books", "youtube", "google videos", "google images", "yahoo", "bing", "ebay", "amazon", "twitter", "baidu", "yandex"];
|
||||
var services = Object.keys(KWS.services);
|
||||
var searches = ["where"];
|
||||
var searchesDifficult = [" * ", "❥"];
|
||||
|
||||
|
||||
@@ -16,6 +16,25 @@ var testData = function () {
|
||||
"twitter.com": "https://twitter.com/i/search/typeahead.json?count=30&result_type=topics&src=SEARCH_BOX&callback=jQuery111&q=I accidentally&_=11111"
|
||||
},
|
||||
responses: {
|
||||
"linkedin": {
|
||||
"resultList": [{
|
||||
"sourceID": "sitefeature",
|
||||
"displayName": "Where is the Answers feature?",
|
||||
"subLine": "",
|
||||
"rank": 0,
|
||||
"id": "162",
|
||||
"url": "http://help.linkedin.com/app/answers/global/id/35227/trk/search-typeahead",
|
||||
"headLine": "<strong>Where<\/strong> is the Answers feature?"
|
||||
}, {
|
||||
"sourceID": "sitefeature",
|
||||
"displayName": "Where are the "Settings" and "Sign Out" links on my homepage?",
|
||||
"subLine": "",
|
||||
"rank": 1,
|
||||
"id": "205",
|
||||
"url": "http://help.linkedin.com/app/answers/global/id/4642/trk/search-typeahead",
|
||||
"headLine": "<strong>Where<\/strong> are the "Settings" and "Sign Out" links on my homepage?"
|
||||
}]
|
||||
},
|
||||
"yahoo": {
|
||||
"gossip": {
|
||||
"qry": "where",
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@
|
||||
<script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js" integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS" crossorigin="anonymous"></script>
|
||||
<script src="//cdn.datatables.net/s/bs/jszip-2.5.0,dt-1.10.10,b-1.1.0,b-colvis-1.1.0,b-html5-1.1.0,b-print-1.1.0,cr-1.3.0,fh-3.1.0,kt-2.1.0,r-2.0.0,se-1.1.0/datatables.min.js" crossorigin="anonymous"></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/4.2.0/lodash.min.js"></script>
|
||||
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
|
||||
<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/FileSaver.js/2014-11-29/FileSaver.min.js"></script>
|
||||
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/localforage/1.3.3/localforage.js"></script>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user