From 5284ccc8b70aaf695eaca83b5050130c862afaf3 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 12 Dec 2016 15:05:19 -0700 Subject: [PATCH 01/18] Added support for the suspect wordlist --- .eslintrc.json | 1 - models/action.js | 11 ++++- models/asset.js | 14 ++++--- models/setting.js | 9 ++++- routes/api/comments/index.js | 14 ++++++- services/wordlist.js | 65 +++++++++++++++++++++--------- tests/routes/api/comments/index.js | 46 ++++++++++++++++++--- tests/services/wordlist.js | 37 ++++++++++------- 8 files changed, 148 insertions(+), 49 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 035a86189..0c31cbb90 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -26,7 +26,6 @@ "yoda": [1], "no-path-concat": [2], "eol-last": [1], - "no-continue": [1], "no-nested-ternary": [1], "no-tabs": [2], "no-unneeded-ternary": [1], diff --git a/models/action.js b/models/action.js index ed18e48c5..b098f871f 100644 --- a/models/action.js +++ b/models/action.js @@ -39,8 +39,17 @@ ActionSchema.statics.findById = function(id) { */ ActionSchema.statics.insertUserAction = (action) => { + // Actions are made unique by using a query that can be reproducable, i.e., + // not containing user inputable values. + let query = { + action_type: action.action_type, + item_type: action.item_type, + item_id: action.item_id, + user_id: action.user_id + }; + // Create/Update the action. - return Action.findOneAndUpdate(action, action, { + return Action.findOneAndUpdate(query, action, { // Ensure that if it's new, we return the new object created. new: true, diff --git a/models/asset.js b/models/asset.js index 3c68b7c13..094f25363 100644 --- a/models/asset.js +++ b/models/asset.js @@ -25,10 +25,6 @@ const AssetSchema = new Schema({ type: Date, default: null }, - settings: { - type: Schema.Types.Mixed, - default: null - }, closedAt: { type: Date, default: null @@ -44,7 +40,15 @@ const AssetSchema = new Schema({ subsection: String, author: String, publication_date: Date, - modified_date: Date + modified_date: Date, + + // This object is used exclusivly for storing settings that are to override + // the base settings from the base Settings object. This is to be accessed + // always after running `rectifySettings` against it. + settings: { + type: Schema.Types.Mixed, + default: null + }, }, { versionKey: false, timestamps: { diff --git a/models/setting.js b/models/setting.js index c7589ca56..d8554bc0d 100644 --- a/models/setting.js +++ b/models/setting.js @@ -3,6 +3,13 @@ const Schema = mongoose.Schema; const _ = require('lodash'); const cache = require('../cache'); +const WordlistSchema = new Schema({ + banned: [String], + suspect: [String] +}, { + _id: false +}); + /** * SettingSchema manages application settings that get used on front and backend. * @type {Schema} @@ -38,7 +45,7 @@ const SettingSchema = new Schema({ type: String, default: '' }, - wordlist: [String] + wordlist: WordlistSchema, }, { timestamps: { createdAt: 'created_at', diff --git a/routes/api/comments/index.js b/routes/api/comments/index.js index 168b89fc3..d0d490bb3 100644 --- a/routes/api/comments/index.js +++ b/routes/api/comments/index.js @@ -76,7 +76,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { // premod, set it to `premod`. let status; - if (req.wordlist.matched) { + if (req.wordlist.banned) { status = Promise.resolve('rejected'); } else { status = Asset @@ -97,7 +97,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { // Return `premod` if pre-moderation is enabled and an empty "new" status // in the event that it is not in pre-moderation mode. - .then(({moderation}) => moderation === 'pre' ? 'premod' : ''); + .then(({moderation}) => moderation === 'pre' ? 'premod' : false); } status.then((status) => Comment.publicCreate({ @@ -108,6 +108,16 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { author_id: req.user.id })) .then((comment) => { + if (req.wordlist.suspect) { + return Comment + .addAction(comment.id, null, 'flag', 'body', 'Matched suspect word filters.') + .then(() => comment); + } + + return comment; + }) + .then((comment) => { + // The comment was created! Send back the created comment. res.status(201).json(comment); }) diff --git a/services/wordlist.js b/services/wordlist.js index 59a842817..2103c7267 100644 --- a/services/wordlist.js +++ b/services/wordlist.js @@ -9,7 +9,10 @@ const Setting = require('../models/setting'); * @type {Object} */ const wordlist = { - list: [], + lists: { + banned: [], + suspect: [] + }, enabled: false }; @@ -32,15 +35,19 @@ wordlist.init = () => { * Inserts the wordlist data and enables the wordlist. * @param {Array} list list of words to be added to the wordlist */ -wordlist.insert = (list) => { +wordlist.insert = (lists) => { // Add the words to this array, but also lowercase the words so that an // easy comparison can take place. - wordlist.list = _.uniq(wordlist.list.concat(list.map((word) => { - return tokenizer.tokenize(word.toLowerCase()); - }))); + ['banned', 'suspect'].forEach((k) => { + if (!(k in lists)) { + return; + } - debug(`Added ${list.length} words to the wordlist, now the wordlist is ${wordlist.list.length} entries long.`); + wordlist.lists[k] = wordlist.parseList(lists[k]); + + debug(`Added ${lists[k].length} words to the ${k} wordlist.`); + }); // Enable the wordlist. wordlist.enabled = true; @@ -48,12 +55,21 @@ wordlist.insert = (list) => { return Promise.resolve(wordlist); }; +/** + * Parses the list content. + * @param {Array} list array of words to parse for a list. + * @return {Array} the parsed list + */ +wordlist.parseList = (list) => _.uniq(list.map((word) => { + return tokenizer.tokenize(word.toLowerCase()); +})); + /** * Tests the phrase to see if it contains any of the defined blockwords. * @param {String} phrase value to check for blockwords. * @return {Boolean} true if a blockword is found, false otherwise. */ -wordlist.match = (phrase) => { +wordlist.match = (list, phrase) => { // Lowercase the word to ensure that we don't miss a match due to // capitalization. @@ -61,7 +77,7 @@ wordlist.match = (phrase) => { // This will return true in the event that at least one blockword is found // in the phrase. - return wordlist.list.some((blockphrase) => { + return list.some((blockphrase) => { // First, let's see if we can find the first word in the blockphrase in the // source phrase. @@ -133,28 +149,39 @@ wordlist.filter = (...fields) => (req, res, next) => { } // Loop over all the fields from the body that we want to check. - const containsProfanity = fields.some((field) => { + for (let i = 0; i < fields.length; i++) { + let field = fields[i]; + let phrase = _.get(req.body, field, false); // If the field doesn't exist in the body, then it can't be profane! if (!phrase) { // Return that there wasn't a profane word here. - return false; + continue; } - // Check if the field contains a profane word. - if (wordlist.match(phrase)) { - debug(`the field "${field}" contained a phrase "${phrase}" which contained a wordlisted word/phrase`); - return true; + // Check if the field contains a banned word. + if (wordlist.match(wordlist.lists.banned, phrase)) { + debug(`the field "${field}" contained a phrase "${phrase}" which contained a banned word/phrase`); + + req.wordlist.banned = ErrContainsProfanity; + + // Stop looping through the fields now, we discovered the worst possible + // situation (a banned word). + break; } - return false; - }); + // Check if the field contains a banned word. + if (wordlist.match(wordlist.lists.suspect, phrase)) { + debug(`the field "${field}" contained a phrase "${phrase}" which contained a suspected word/phrase`); - // The body could contain some profanity, address that here. - if (containsProfanity) { - req.wordlist.matched = ErrContainsProfanity; + req.wordlist.suspect = ErrContainsProfanity; + + // Continue looping through the fields now, we discovered a possible bad + // word (suspect). + continue; + } } next(); diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index b3e4e2675..c720130c0 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -21,7 +21,7 @@ describe('/api/v1/comments', () => { // Ensure that the settings are always available. beforeEach(() => Promise.all([ - wordlist.insert(['bad words']), + wordlist.insert({banned: ['bad words'], suspect: ['suspect words']}), Setting.init(settings) ])); @@ -145,12 +145,22 @@ describe('/api/v1/comments', () => { describe('#post', () => { let asset_id; + let postmod_asset_id; - beforeEach(() => Asset.findOrCreateByUrl('https://coralproject.net/section/article-is-the-best').then((asset) => { + beforeEach(() => Promise.all([ + Asset.findOrCreateByUrl('https://coralproject.net/section/article-is-the-best').then((asset) => { - // Update the asset id. - asset_id = asset.id; - })); + // Update the asset id. + asset_id = asset.id; + }), + Asset.findOrCreateByUrl('https://coralproject.net/section/postmod-article-is-the-best').then((asset) => { + + // Update the asset id. + postmod_asset_id = asset.id; + + return Asset.overrideSettings(postmod_asset_id, {moderation: 'post'}); + }), + ])); it('should create a comment', () => { return chai.request(app) @@ -175,6 +185,32 @@ describe('/api/v1/comments', () => { }); }); + it('should create a comment with no status and a flag if it contains a suspected word', () => { + return chai.request(app) + .post('/api/v1/comments') + .set(passport.inject({roles: []})) + .send({'body': 'suspect words are the most suspicious', 'author_id': '123', 'asset_id': postmod_asset_id, 'parent_id': ''}) + .then((res) => { + expect(res).to.have.status(201); + expect(res.body).to.have.property('id'); + expect(res.body).to.have.property('status', null); + + return Promise.all([ + res.body, + Action.findByType('flag', 'comments') + ]); + }) + .then(([comment, actions]) => { + expect(actions).to.have.length(1); + + let action = actions[0]; + + expect(action).to.have.property('item_id', comment.id); + expect(action).to.have.property('field', 'body'); + expect(action).to.have.property('detail', 'Matched suspect word filters.'); + }); + }); + it('should create a comment with a premod status if it\'s asset is has pre-moderation enabled', () => { return Asset .findOrCreateByUrl('https://coralproject.net/article1') diff --git a/tests/services/wordlist.js b/tests/services/wordlist.js index 0ae76c176..c65be3b11 100644 --- a/tests/services/wordlist.js +++ b/tests/services/wordlist.js @@ -4,22 +4,25 @@ const wordlist = require('../../services/wordlist'); describe('wordlist: services', () => { - before(() => wordlist.insert([ - 'BAD', - 'bad', - 'how to murder', - 'how to kill' - ])); - - beforeEach(() => { - expect(wordlist.list).to.not.be.empty; - expect(wordlist.enabled).to.be.true; - }); + const wordlists = { + banned: [ + 'BAD', + 'bad', + 'how to murder', + 'how to kill' + ], + suspect: [ + 'murder' + ] + }; describe('#init', () => { + before(() => wordlist.insert(wordlists)); + it('has entries', () => { - expect(wordlist.list).to.not.be.empty; + expect(wordlist.lists.banned).to.not.be.empty; + expect(wordlist.lists.suspect).to.not.be.empty; expect(wordlist.enabled).to.be.true; }); @@ -27,6 +30,8 @@ describe('wordlist: services', () => { describe('#match', () => { + const bannedList = wordlist.parseList(wordlists.banned); + it('does match on a bad word', () => { [ 'how to kill', @@ -36,7 +41,7 @@ describe('wordlist: services', () => { 'how to murder', 'How To mUrDer' ].forEach((word) => { - expect(wordlist.match(word)).to.be.true; + expect(wordlist.match(bannedList, word)).to.be.true; }); }); @@ -48,7 +53,7 @@ describe('wordlist: services', () => { 'how to be a great person?', 'how to not kill?' ].forEach((word) => { - expect(wordlist.match(word)).to.be.false; + expect(wordlist.match(bannedList, word)).to.be.false; }); }); @@ -56,6 +61,8 @@ describe('wordlist: services', () => { describe('#filter', () => { + before(() => wordlist.insert(wordlists)); + it('matches on bodies containing bad words', (done) => { let req = { @@ -68,7 +75,7 @@ describe('wordlist: services', () => { expect(err).to.be.undefined; expect(req).to.have.property('wordlist'); expect(req.wordlist).to.have.property('matched'); - expect(req.wordlist.matched).to.be.equal(wordlist.ErrContainsProfanity); + expect(req.wordlist.banned).to.be.equal(wordlist.ErrContainsProfanity); done(); }); From 7742dbf9a202479051b56179534388a8c76dbc98 Mon Sep 17 00:00:00 2001 From: Riley Davis Date: Mon, 19 Dec 2016 13:07:50 -0700 Subject: [PATCH 02/18] add a suspect words card --- .../containers/Configure/CommentSettings.js | 201 +++++++++--------- .../src/containers/Configure/Configure.css | 4 +- .../src/containers/Configure/Configure.js | 23 +- .../src/containers/Configure/EmbedLink.js | 25 ++- .../src/containers/Configure/Wordlist.js | 36 +++- client/coral-admin/src/translations.json | 12 +- 6 files changed, 166 insertions(+), 135 deletions(-) diff --git a/client/coral-admin/src/containers/Configure/CommentSettings.js b/client/coral-admin/src/containers/Configure/CommentSettings.js index 050432743..7fe0a7938 100644 --- a/client/coral-admin/src/containers/Configure/CommentSettings.js +++ b/client/coral-admin/src/containers/Configure/CommentSettings.js @@ -69,109 +69,114 @@ const updateClosedTimeout = (updateSettings, ts, isMeasure) => (event) => { } }; -const CommentSettings = ({fetchingSettings, updateSettings, settingsError, settings, errors}) => { +const CommentSettings = ({fetchingSettings, title, updateSettings, settingsError, settings, errors}) => { if (fetchingSettings) { /* maybe a spinner here at some point */ return

Loading settings...

; } - return - - - - - -
{lang.t('configure.enable-pre-moderation')}
-

- {lang.t('configure.enable-pre-moderation-text')} -

- - - - - - - -
{lang.t('configure.comment-count-header')}
-

- {lang.t('configure.comment-count-text-pre')} - - {lang.t('configure.comment-count-text-post')} - { - errors.charCount && - -
- - {lang.t('configure.comment-count-error')} - - } -

- - - - - - - - {lang.t('configure.include-comment-stream')} -

- {lang.t('configure.include-comment-stream-desc')} -

- - - - - - - - - - {lang.t('configure.close-after')} -
- -
- - - - - -
- - - - - {lang.t('configure.closed-comments-desc')} - - - - ; + return ( +
+

{title}

+ + + + + + +
{lang.t('configure.enable-pre-moderation')}
+

+ {lang.t('configure.enable-pre-moderation-text')} +

+ + + + + + + +
{lang.t('configure.comment-count-header')}
+

+ {lang.t('configure.comment-count-text-pre')} + + {lang.t('configure.comment-count-text-post')} + { + errors.charCount && + +
+ + {lang.t('configure.comment-count-error')} + + } +

+ + + + + + + + {lang.t('configure.include-comment-stream')} +

+ {lang.t('configure.include-comment-stream-desc')} +

+ + + + + + + + + + {lang.t('configure.close-after')} +
+ +
+ + + + + +
+ + + + + {lang.t('configure.closed-comments-desc')} + + + + +
+ ); }; export default CommentSettings; diff --git a/client/coral-admin/src/containers/Configure/Configure.css b/client/coral-admin/src/containers/Configure/Configure.css index c0646c9e2..2b5c8d578 100644 --- a/client/coral-admin/src/containers/Configure/Configure.css +++ b/client/coral-admin/src/containers/Configure/Configure.css @@ -112,12 +112,12 @@ letter-spacing: 0.03em; } -#bannedWordlist { +#bannedWordlist, #suspectWordlist { width: 100%; padding: 10px; } -.bannedWordHeader { +.wordlistHeader { font-weight: bold; font-size:18px; margin-bottom:3px; diff --git a/client/coral-admin/src/containers/Configure/Configure.js b/client/coral-admin/src/containers/Configure/Configure.js index ed8ba9b26..6af02cf05 100644 --- a/client/coral-admin/src/containers/Configure/Configure.js +++ b/client/coral-admin/src/containers/Configure/Configure.js @@ -36,7 +36,8 @@ class Configure extends React.Component { || !this.props.settings.wordlist) && newProps.settings.wordlist && newProps.settings.wordlist.length !== 0 ) { - this.setState({wordlist: newProps.settings.wordlist.join(', ')}); + console.log('wordlist?', newProps.settings.wordlist); + this.setState({wordlist: newProps.settings.wordlist.banned.join(', ')}); } } @@ -74,46 +75,43 @@ class Configure extends React.Component { }); } - getSection = (section) => { + getSection (section) { + const pageTitle = this.getPageTitle(section); switch(section){ case 'comments': return ; case 'embed': - return ; + return ; case 'wordlist': return ; } } - getPageTitle = (section) => { + getPageTitle (section) { switch(section) { case 'comments': return lang.t('configure.comment-settings'); case 'embed': return lang.t('configure.embed-comment-stream'); - case 'wordlist': - return lang.t('configure.wordlist'); + default: + return ''; } } render () { - let pageTitle = this.getPageTitle(this.state.activeSection); const section = this.getSection(this.state.activeSection); const showSave = Object.keys(this.state.errors).reduce( (bool, error) => this.state.errors[error] ? false : bool, this.state.changed); - if (this.props.fetchingSettings) { - pageTitle += ' - Loading...'; - } - return (
@@ -151,7 +149,6 @@ class Configure extends React.Component {
-

{pageTitle}

{ this.props.saveFetchingError } { this.props.fetchSettingsError } { section } diff --git a/client/coral-admin/src/containers/Configure/EmbedLink.js b/client/coral-admin/src/containers/Configure/EmbedLink.js index 6c5c3da99..374d78ea3 100644 --- a/client/coral-admin/src/containers/Configure/EmbedLink.js +++ b/client/coral-admin/src/containers/Configure/EmbedLink.js @@ -31,16 +31,21 @@ class EmbedLink extends Component { render () { const embedText = `
`; - return - -

{lang.t('configure.copy-and-paste')}

-