wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-07-19 11:28:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reimplement wordlist on server using same regexp solution

Browse Source
This commit is contained in:
Chi Vinh Le
2017-09-14 17:36:04 +07:00
parent abc651c55d
commit ad361413c7
3 changed files with 45 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/coral-admin/src/components/CommentBodyHighlighter.js
+1
View File
@@ -6,6 +6,7 @@ function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}
// generate a regulare expression that catches the `phrases`.
function generateRegExp(phrases) {
const inner = phrases
.map((phrase) =>
services/wordlist.js
+41 -103
View File
@@ -1,13 +1,39 @@
const debug = require('debug')('talk:services:wordlist');
const _ = require('lodash');
const {RegexpTokenizer} = require('natural');
const tokenizer = new RegexpTokenizer({pattern: /[.\s'"?!]/});
const nameTokenizer = new RegexpTokenizer({pattern: /_/});
const SettingsService = require('./settings');
const Errors = require('../errors');
const memoize = require('lodash/memoize');
// REGEX to prevent emoji's from entering the wordlist.
const EMOJI_REGEX = /(?:[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff])[\ufe0e\ufe0f]?(?:[\u0300-\u036f\ufe20-\ufe23\u20d0-\u20f0]|\ud83c[\udffb-\udfff])?(?:\u200d(?:[^\ud800-\udfff]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff])[\ufe0e\ufe0f]?(?:[\u0300-\u036f\ufe20-\ufe23\u20d0-\u20f0]|\ud83c[\udffb-\udfff])?)*/;
/**
* Escape string for special regular expression characters.
*/
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
}
/**
* Generate a regulare expression that catches the `phrases`.
*/
function generateRegExp(phrases) {
const inner = phrases
.map((phrase) =>
phrase.split(/\s+/)
.map((word) => escapeRegExp(word))
.join('[\\s"?!.]+')
).join('|');
return new RegExp(`(^|[^\\w])(${inner})(?=[^\\w]|$)`, 'iu');
}
/**
* Memoized version of generateRegExp.
*/
const generateRegExpMemoized = memoize(generateRegExp, (phrases) => phrases.join(','));
/**
* Never matching regexp that exits immediately.
*/
const neverMatch = /(?!)/;
/**
* The root wordlist object.
@@ -16,9 +42,9 @@ const EMOJI_REGEX = /(?:[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\ud
class Wordlist {
constructor() {
this.lists = {
banned: [],
suspect: []
this.regexp = {
banned: neverMatch,
suspect: neverMatch,
};
}
@@ -48,7 +74,9 @@ class Wordlist {
return;
}
this.lists[k] = Wordlist.parseList(lists[k]);
this.regexp[k] = lists[k] && lists[k].length > 0
? generateRegExpMemoized(lists[k])
: neverMatch;
debug(`Added ${lists[k].length} words to the ${k} wordlist.`);
});
@@ -56,92 +84,6 @@ class Wordlist {
return Promise.resolve(this);
}
/**
* Parses the list content.
* @param {Array} list array of words to parse for a list.
* @return {Array} the parsed list
*/
static parseList(list) {
return _.uniq(list.filter((word) => {
if (EMOJI_REGEX.test(word)) {
return false;
}
return true;
})
.map((word) => {
if (word.length === 1) {
return [word];
}
return tokenizer.tokenize(word.toLowerCase());
})
.filter((tokens) => {
if (tokens.length === 0) {
return false;
}
return true;
}));
}
/**
* Tests the phrase to see if it contains any of the defined blockwords.
* @param {String} phrase value to check for blockwords.
* @return {Boolean} true if a blockword is found, false otherwise.
*/
match(list, phrase, tk = tokenizer) {
// Lowercase the word to ensure that we don't miss a match due to
// capitalization.
let lowerPhraseWords = tk.tokenize(phrase.toLowerCase());
// This will return true in the event that at least one blockword is found
// in the phrase.
return list.some((blockphrase) => {
// First, let's see if we can find the first word in the blockphrase in the
// source phrase.
let idx = lowerPhraseWords.indexOf(blockphrase[0]);
if (idx === -1) {
// The first blockword in the blockphrase did not match the source phrase
// anywhere.
return false;
}
// Here we'll quick respond with true in the event that the blockphrase was
// just a single word.
if (blockphrase.length === 1) {
return true;
}
// We found the first word in the source phrase! Lets ensure it matches the
// rest of the blockphrase...
// Check to see if it even has the length to support this word!
if (lowerPhraseWords.length < idx + blockphrase.length - 1) {
// We couldn't possibly have the entire phrase here because we don't have
// enough entries!
return false;
}
for (let i = 1; i < blockphrase.length; i++) {
// Check to see if the next word also matches!
if (lowerPhraseWords[idx + i] !== blockphrase[i]) {
return false;
}
}
// We've walked over all the words of the blockphrase, and haven't had a
// mismatch... It does contain the whole word!
return true;
});
}
/**
* Scans a specific field for wordlist violations.
*/
@@ -156,7 +98,7 @@ class Wordlist {
}
// Check if the field contains a banned word.
if (this.match(this.lists.banned, phrase)) {
if (this.regexp.banned.test(phrase)) {
debug(`the field "${fieldName}" contained a phrase "${phrase}" which contained a banned word/phrase`);
errors.banned = Errors.ErrContainsProfanity;
@@ -166,8 +108,8 @@ class Wordlist {
return errors;
}
// Check if the field contains a banned word.
if (this.match(this.lists.suspect, phrase)) {
// Check if the field contains a suspected word.
if (this.regexp.suspect.test(phrase)) {
debug(`the field "${fieldName}" contained a phrase "${phrase}" which contained a suspected word/phrase`);
errors.suspect = Errors.ErrContainsProfanity;
@@ -231,16 +173,12 @@ class Wordlist {
return wl
.load()
.then(() => {
if (!wl.checkName(wl.lists.banned, username)) {
if (wl.regexp.banned.test(username)) {
return Errors.ErrContainsProfanity;
}
});
}
checkName(list, name) {
return !this.match(list, name, nameTokenizer);
}
/**
* Connect middleware for scanning request bodies for wordlisted words and
* attaching a ErrContainsProfanity to the req.wordlisted parameter, otherwise
test/server/services/wordlist.js
+3 -57
View File
@@ -27,44 +27,10 @@ describe('services.Wordlist', () => {
beforeEach(() => SettingsService.init(settings));
describe('#init', () => {
describe('#regexp', () => {
before(() => wordlist.upsert(wordlists));
it('parses the wordlists correctly', () => {
expect(wordlist.lists.banned).to.deep.equal([
[ 'cookies' ],
[ 'how', 'to', 'do', 'bad', 'things' ],
[ 'how', 'to', 'do', 'really', 'bad', 'things' ],
[ 's', 'h', 'i', 't' ],
[ '$hit' ],
[ 'p**ch' ],
[ 'p*ch' ],
]);
expect(wordlist.lists.suspect).to.deep.equal([
[ 'do', 'bad', 'things' ],
]);
});
});
describe('#parseList', () => {
it('does not include emojis in the wordlist', () => {
let list = Wordlist.parseList([
'🖕',
'🖕 asdf',
'asd🖕asdf',
'asd🖕',
]);
expect(list).to.have.length(0);
});
});
const bannedList = Wordlist.parseList(wordlists.banned);
describe('#match', () => {
it('does match on a bad word', () => {
[
'how to do really bad things',
@@ -76,7 +42,7 @@ describe('services.Wordlist', () => {
'This stuff is $hit!',
'That\'s a p**ch!',
].forEach((word) => {
expect(wordlist.match(bannedList, word)).to.be.true;
expect(wordlist.regexp.banned.test(word)).to.be.true;
});
});
@@ -90,7 +56,7 @@ describe('services.Wordlist', () => {
'I have bad $ hit lling',
'That\'s a p***ch!',
].forEach((word) => {
expect(wordlist.match(bannedList, word)).to.be.false;
expect(wordlist.regexp.banned.test(word)).to.be.false;
});
});
@@ -129,26 +95,6 @@ describe('services.Wordlist', () => {
});
describe('#checkName', () => {
[
'flowers',
'joy',
'lots_of_candy'
].forEach((username) => {
it(`does not match on list=banned name=${username}`, () => {
expect(wordlist.checkName(bannedList, username)).to.be.true;
});
});
[
'cookies'
].forEach((username) => {
it(`does match on list=banned name=${username}`, () => {
expect(wordlist.checkName(bannedList, username)).to.be.false;
});
});
});
describe('#filter', () => {
before(() => wordlist.upsert(wordlists));