adjusted tokenizer to use regex based tokenizer instead

services/wordlist.js

@@ -1,8 +1,8 @@
 const debug = require('debug')('talk:services:wordlist');
 const _ = require('lodash');
-const natural = require('natural');
-const tokenizer = new natural.WordTokenizer();
-const nameTokenizer = new natural.RegexpTokenizer({pattern: /\_/});
+const {RegexpTokenizer} = require('natural');
+const tokenizer = new RegexpTokenizer({pattern: /[\.\s\'\"\?\!]/});
+const nameTokenizer = new RegexpTokenizer({pattern: /\_/});
 const SettingsService = require('./settings');
 const Errors = require('../errors');
 
@@ -73,7 +73,7 @@ class Wordlist {
       if (word.length === 1) {
         return [word];
       }
-      
+
       return tokenizer.tokenize(word.toLowerCase());
     })
     .filter((tokens) => {

test/server/services/wordlist.js

@@ -10,10 +10,12 @@ describe('services.Wordlist', () => {
       'cookies',
       'how to do bad things',
       'how to do really bad things',
-      's h i t'
+      's h i t',
+      '$hit',
+      'p**ch',
     ],
     suspect: [
-      'do bad things'
+      'do bad things',
     ]
   };
 
@@ -26,9 +28,18 @@ describe('services.Wordlist', () => {
 
     before(() => wordlist.upsert(wordlists));
 
-    it('has entries', () => {
-      expect(wordlist.lists.banned).to.not.be.empty;
-      expect(wordlist.lists.suspect).to.not.be.empty;
+    it('parses the wordlists correctly', () => {
+      expect(wordlist.lists.banned).to.deep.equal([
+        [ 'cookies' ],
+        [ 'how', 'to', 'do', 'bad', 'things' ],
+        [ 'how', 'to', 'do', 'really', 'bad', 'things' ],
+        [ 's', 'h', 'i', 't' ],
+        [ '$hit' ],
+        [ 'p**ch' ],
+      ]);
+      expect(wordlist.lists.suspect).to.deep.equal([
+        [ 'do', 'bad', 'things' ],
+      ]);
     });
 
   });
@@ -57,7 +68,9 @@ describe('services.Wordlist', () => {
         'cookies',
         'COOKIES.',
         'how to do bad things',
-        'How To do bad things!'
+        'How To do bad things!',
+        'This stuff is $hit!',
+        'That\'s a p**ch!',
       ].forEach((word) => {
         expect(wordlist.match(bannedList, word)).to.be.true;
       });
@@ -68,7 +81,9 @@ describe('services.Wordlist', () => {
         'how to',
         'cookie',
         'how to be a great person?',
-        'how to not do really bad things?'
+        'how to not do really bad things?',
+        'i have $100 dollars.',
+        'I have bad $ hit lling',
       ].forEach((word) => {
         expect(wordlist.match(bannedList, word)).to.be.false;
       });