Added support for the suspect wordlist

.eslintrc.json

@@ -26,7 +26,6 @@
         "yoda": [1],
         "no-path-concat": [2],
         "eol-last": [1],
-        "no-continue": [1],
         "no-nested-ternary": [1],
         "no-tabs": [2],
         "no-unneeded-ternary": [1],

models/action.js

@@ -39,8 +39,17 @@ ActionSchema.statics.findById = function(id) {
  */
 ActionSchema.statics.insertUserAction = (action) => {
 
+  // Actions are made unique by using a query that can be reproducable, i.e.,
+  // not containing user inputable values.
+  let query = {
+    action_type: action.action_type,
+    item_type: action.item_type,
+    item_id: action.item_id,
+    user_id: action.user_id
+  };
+
   // Create/Update the action.
-  return Action.findOneAndUpdate(action, action, {
+  return Action.findOneAndUpdate(query, action, {
 
     // Ensure that if it's new, we return the new object created.
     new: true,

models/asset.js

@@ -25,10 +25,6 @@ const AssetSchema = new Schema({
     type: Date,
     default: null
   },
-  settings: {
-    type: Schema.Types.Mixed,
-    default: null
-  },
   closedAt: {
     type: Date,
     default: null
@@ -44,7 +40,15 @@ const AssetSchema = new Schema({
   subsection: String,
   author: String,
   publication_date: Date,
-  modified_date: Date
+  modified_date: Date,
+
+  // This object is used exclusivly for storing settings that are to override
+  // the base settings from the base Settings object. This is to be accessed
+  // always after running `rectifySettings` against it.
+  settings: {
+    type: Schema.Types.Mixed,
+    default: null
+  },
 }, {
   versionKey: false,
   timestamps: {

models/setting.js

@@ -3,6 +3,13 @@ const Schema = mongoose.Schema;
 const _ = require('lodash');
 const cache = require('../cache');
 
+const WordlistSchema = new Schema({
+  banned: [String],
+  suspect: [String]
+}, {
+  _id: false
+});
+
 /**
  * SettingSchema manages application settings that get used on front and backend.
  * @type {Schema}
@@ -38,7 +45,7 @@ const SettingSchema = new Schema({
     type: String,
     default: ''
   },
-  wordlist: [String]
+  wordlist: WordlistSchema,
 }, {
   timestamps: {
     createdAt: 'created_at',

routes/api/comments/index.js

@@ -76,7 +76,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
   // premod, set it to `premod`.
   let status;
 
-  if (req.wordlist.matched) {
+  if (req.wordlist.banned) {
     status = Promise.resolve('rejected');
   } else {
     status = Asset
@@ -97,7 +97,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
 
       // Return `premod` if pre-moderation is enabled and an empty "new" status
       // in the event that it is not in pre-moderation mode.
-      .then(({moderation}) => moderation === 'pre' ? 'premod' : '');
+      .then(({moderation}) => moderation === 'pre' ? 'premod' : false);
   }
 
   status.then((status) => Comment.publicCreate({
@@ -108,6 +108,16 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
     author_id: req.user.id
   }))
   .then((comment) => {
+    if (req.wordlist.suspect) {
+      return Comment
+        .addAction(comment.id, null, 'flag', 'body', 'Matched suspect word filters.')
+        .then(() => comment);
+    }
+
+    return comment;
+  })
+  .then((comment) => {
+
     // The comment was created! Send back the created comment.
     res.status(201).json(comment);
   })

services/wordlist.js

@@ -9,7 +9,10 @@ const Setting = require('../models/setting');
  * @type {Object}
  */
 const wordlist = {
-  list: [],
+  lists: {
+    banned: [],
+    suspect: []
+  },
   enabled: false
 };
 
@@ -32,15 +35,19 @@ wordlist.init = () => {
  * Inserts the wordlist data and enables the wordlist.
  * @param  {Array} list list of words to be added to the wordlist
  */
-wordlist.insert = (list) => {
+wordlist.insert = (lists) => {
 
   // Add the words to this array, but also lowercase the words so that an
   // easy comparison can take place.
-  wordlist.list = _.uniq(wordlist.list.concat(list.map((word) => {
-    return tokenizer.tokenize(word.toLowerCase());
-  })));
+  ['banned', 'suspect'].forEach((k) => {
+    if (!(k in lists)) {
+      return;
+    }
 
-  debug(`Added ${list.length} words to the wordlist, now the wordlist is ${wordlist.list.length} entries long.`);
+    wordlist.lists[k] = wordlist.parseList(lists[k]);
+
+    debug(`Added ${lists[k].length} words to the ${k} wordlist.`);
+  });
 
   // Enable the wordlist.
   wordlist.enabled = true;
@@ -48,12 +55,21 @@ wordlist.insert = (list) => {
   return Promise.resolve(wordlist);
 };
 
+/**
+ * Parses the list content.
+ * @param  {Array} list array of words to parse for a list.
+ * @return {Array}      the parsed list
+ */
+wordlist.parseList = (list) => _.uniq(list.map((word) => {
+  return tokenizer.tokenize(word.toLowerCase());
+}));
+
 /**
  * Tests the phrase to see if it contains any of the defined blockwords.
  * @param  {String} phrase value to check for blockwords.
  * @return {Boolean}       true if a blockword is found, false otherwise.
  */
-wordlist.match = (phrase) => {
+wordlist.match = (list, phrase) => {
 
   // Lowercase the word to ensure that we don't miss a match due to
   // capitalization.
@@ -61,7 +77,7 @@ wordlist.match = (phrase) => {
 
   // This will return true in the event that at least one blockword is found
   // in the phrase.
-  return wordlist.list.some((blockphrase) => {
+  return list.some((blockphrase) => {
 
     // First, let's see if we can find the first word in the blockphrase in the
     // source phrase.
@@ -133,28 +149,39 @@ wordlist.filter = (...fields) => (req, res, next) => {
   }
 
   // Loop over all the fields from the body that we want to check.
-  const containsProfanity = fields.some((field) => {
+  for (let i = 0; i < fields.length; i++) {
+    let field = fields[i];
+
     let phrase = _.get(req.body, field, false);
 
     // If the field doesn't exist in the body, then it can't be profane!
     if (!phrase) {
 
       // Return that there wasn't a profane word here.
-      return false;
+      continue;
     }
 
-    // Check if the field contains a profane word.
-    if (wordlist.match(phrase)) {
-      debug(`the field "${field}" contained a phrase "${phrase}" which contained a wordlisted word/phrase`);
-      return true;
+    // Check if the field contains a banned word.
+    if (wordlist.match(wordlist.lists.banned, phrase)) {
+      debug(`the field "${field}" contained a phrase "${phrase}" which contained a banned word/phrase`);
+
+      req.wordlist.banned = ErrContainsProfanity;
+
+      // Stop looping through the fields now, we discovered the worst possible
+      // situation (a banned word).
+      break;
     }
 
-    return false;
-  });
+    // Check if the field contains a banned word.
+    if (wordlist.match(wordlist.lists.suspect, phrase)) {
+      debug(`the field "${field}" contained a phrase "${phrase}" which contained a suspected word/phrase`);
 
-  // The body could contain some profanity, address that here.
-  if (containsProfanity) {
-    req.wordlist.matched = ErrContainsProfanity;
+      req.wordlist.suspect = ErrContainsProfanity;
+
+      // Continue looping through the fields now, we discovered a possible bad
+      // word (suspect).
+      continue;
+    }
   }
 
   next();

tests/routes/api/comments/index.js

@@ -21,7 +21,7 @@ describe('/api/v1/comments', () => {
 
   // Ensure that the settings are always available.
   beforeEach(() => Promise.all([
-    wordlist.insert(['bad words']),
+    wordlist.insert({banned: ['bad words'], suspect: ['suspect words']}),
     Setting.init(settings)
   ]));
 
@@ -145,12 +145,22 @@ describe('/api/v1/comments', () => {
   describe('#post', () => {
 
     let asset_id;
+    let postmod_asset_id;
 
-    beforeEach(() => Asset.findOrCreateByUrl('https://coralproject.net/section/article-is-the-best').then((asset) => {
+    beforeEach(() => Promise.all([
+      Asset.findOrCreateByUrl('https://coralproject.net/section/article-is-the-best').then((asset) => {
 
-      // Update the asset id.
-      asset_id = asset.id;
-    }));
+        // Update the asset id.
+        asset_id = asset.id;
+      }),
+      Asset.findOrCreateByUrl('https://coralproject.net/section/postmod-article-is-the-best').then((asset) => {
+
+        // Update the asset id.
+        postmod_asset_id = asset.id;
+
+        return Asset.overrideSettings(postmod_asset_id, {moderation: 'post'});
+      }),
+    ]));
 
     it('should create a comment', () => {
       return chai.request(app)
@@ -175,6 +185,32 @@ describe('/api/v1/comments', () => {
         });
     });
 
+    it('should create a comment with no status and a flag if it contains a suspected word', () => {
+      return chai.request(app)
+        .post('/api/v1/comments')
+        .set(passport.inject({roles: []}))
+        .send({'body': 'suspect words are the most suspicious', 'author_id': '123', 'asset_id': postmod_asset_id, 'parent_id': ''})
+        .then((res) => {
+          expect(res).to.have.status(201);
+          expect(res.body).to.have.property('id');
+          expect(res.body).to.have.property('status', null);
+
+          return Promise.all([
+            res.body,
+            Action.findByType('flag', 'comments')
+          ]);
+        })
+        .then(([comment, actions]) => {
+          expect(actions).to.have.length(1);
+
+          let action = actions[0];
+
+          expect(action).to.have.property('item_id', comment.id);
+          expect(action).to.have.property('field', 'body');
+          expect(action).to.have.property('detail', 'Matched suspect word filters.');
+        });
+    });
+
     it('should create a comment with a premod status if it\'s asset is has pre-moderation enabled', () => {
       return Asset
         .findOrCreateByUrl('https://coralproject.net/article1')

tests/services/wordlist.js

@@ -4,22 +4,25 @@ const wordlist = require('../../services/wordlist');
 
 describe('wordlist: services', () => {
 
-  before(() => wordlist.insert([
-    'BAD',
-    'bad',
-    'how to murder',
-    'how to kill'
-  ]));
-
-  beforeEach(() => {
-    expect(wordlist.list).to.not.be.empty;
-    expect(wordlist.enabled).to.be.true;
-  });
+  const wordlists = {
+    banned: [
+      'BAD',
+      'bad',
+      'how to murder',
+      'how to kill'
+    ],
+    suspect: [
+      'murder'
+    ]
+  };
 
   describe('#init', () => {
 
+    before(() => wordlist.insert(wordlists));
+
     it('has entries', () => {
-      expect(wordlist.list).to.not.be.empty;
+      expect(wordlist.lists.banned).to.not.be.empty;
+      expect(wordlist.lists.suspect).to.not.be.empty;
       expect(wordlist.enabled).to.be.true;
     });
 
@@ -27,6 +30,8 @@ describe('wordlist: services', () => {
 
   describe('#match', () => {
 
+    const bannedList = wordlist.parseList(wordlists.banned);
+
     it('does match on a bad word', () => {
       [
         'how to kill',
@@ -36,7 +41,7 @@ describe('wordlist: services', () => {
         'how to murder',
         'How To mUrDer'
       ].forEach((word) => {
-        expect(wordlist.match(word)).to.be.true;
+        expect(wordlist.match(bannedList, word)).to.be.true;
       });
     });
 
@@ -48,7 +53,7 @@ describe('wordlist: services', () => {
         'how to be a great person?',
         'how to not kill?'
       ].forEach((word) => {
-        expect(wordlist.match(word)).to.be.false;
+        expect(wordlist.match(bannedList, word)).to.be.false;
       });
     });
 
@@ -56,6 +61,8 @@ describe('wordlist: services', () => {
 
   describe('#filter', () => {
 
+    before(() => wordlist.insert(wordlists));
+
     it('matches on bodies containing bad words', (done) => {
 
       let req = {
@@ -68,7 +75,7 @@ describe('wordlist: services', () => {
         expect(err).to.be.undefined;
         expect(req).to.have.property('wordlist');
         expect(req.wordlist).to.have.property('matched');
-        expect(req.wordlist.matched).to.be.equal(wordlist.ErrContainsProfanity);
+        expect(req.wordlist.banned).to.be.equal(wordlist.ErrContainsProfanity);
 
         done();
       });