mirror of
https://github.com/wassname/talk.git
synced 2026-07-11 12:36:22 +08:00
revamped configuration for moderation phases
This commit is contained in:
Wyatt Johnson
+213
-84
@@ -7,6 +7,7 @@ const TagsService = require('../../services/tags');
|
||||
const CommentsService = require('../../services/comments');
|
||||
const KarmaService = require('../../services/karma');
|
||||
const tlds = require('tlds');
|
||||
const merge = require('lodash/merge');
|
||||
const linkify = require('linkify-it')()
|
||||
.tlds(tlds);
|
||||
const Wordlist = require('../../services/wordlist');
|
||||
@@ -156,7 +157,7 @@ const adjustKarma = (Comments, id, status) => async () => {
|
||||
* @param {String} [status='NONE'] the status of the new comment
|
||||
* @return {Promise} resolves to the created comment
|
||||
*/
|
||||
const createComment = async (context, {tags = [], body, asset_id, parent_id = null, metadata = {}}, status = 'NONE') => {
|
||||
const createComment = async (context, {tags = [], body, asset_id, parent_id = null, status = 'NONE', metadata = {}}) => {
|
||||
const {user, loaders: {Comments}, pubsub} = context;
|
||||
|
||||
// Resolve the tags for the comment.
|
||||
@@ -224,6 +225,169 @@ const filterNewComment = async (context, {body, asset_id}) => {
|
||||
];
|
||||
};
|
||||
|
||||
const moderationPhases = [
|
||||
|
||||
// This phase checks to see if the comment is long enough.
|
||||
(context, comment) => {
|
||||
|
||||
// Check to see if the body is too short, if it is, then complain about it!
|
||||
if (comment.body.length < 2) {
|
||||
throw errors.ErrCommentTooShort;
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks to see if the asset being processed is closed or not.
|
||||
(context, comment, {asset}) => {
|
||||
|
||||
// Check to see if the asset has closed commenting...
|
||||
if (asset.isClosed) {
|
||||
throw new errors.ErrAssetCommentingClosed(asset.closedMessage);
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks the comment against the wordlist.
|
||||
(context, comment, {wordlist}) => {
|
||||
|
||||
// Decide the status based on whether or not the current asset/settings
|
||||
// has pre-mod enabled or not. If the comment was rejected based on the
|
||||
// wordlist, then reject it, otherwise if the moderation setting is
|
||||
// premod, set it to `premod`.
|
||||
if (wordlist.banned) {
|
||||
|
||||
// Add the flag related to Trust to the comment.
|
||||
return {
|
||||
status: 'REJECTED',
|
||||
actions: [{
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'BANNED_WORD',
|
||||
metadata: {}
|
||||
}]
|
||||
};
|
||||
}
|
||||
|
||||
// If the comment has a suspect word or a link, we need to add a
|
||||
// flag to it to indicate that it needs to be looked at.
|
||||
// Otherwise just return the new comment.
|
||||
|
||||
// If the wordlist has matched the suspect word filter and we haven't disabled
|
||||
// auto-flagging suspect words, then we should flag the comment!
|
||||
if (wordlist.suspect && !DISABLE_AUTOFLAG_SUSPECT_WORDS) {
|
||||
|
||||
// TODO: this is kind of fragile, we should refactor this to resolve
|
||||
// all these const's that we're using like 'COMMENTS', 'FLAG' to be
|
||||
// defined in a checkable schema.
|
||||
return {
|
||||
actions: [{
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'Matched suspect word filter',
|
||||
metadata: {}
|
||||
}],
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks to see if the comment's length exeeds maximum.
|
||||
(context, comment, {assetSettings: {charCountEnable, charCount}}) => {
|
||||
|
||||
// Reject if the comment is too long
|
||||
if (charCountEnable && comment.body.length > charCount) {
|
||||
|
||||
// Add the flag related to Trust to the comment.
|
||||
return {
|
||||
status: 'REJECTED',
|
||||
actions: [{
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'BODY_COUNT',
|
||||
metadata: {
|
||||
count: comment.body.length,
|
||||
}
|
||||
}]
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks the comment if it has any links in it if the check is
|
||||
// enabled.
|
||||
(context, comment, {assetSettings: {premodLinksEnable}}) => {
|
||||
if (premodLinksEnable && linkify.test(comment.body)) {
|
||||
|
||||
// Add the flag related to Trust to the comment.
|
||||
return {
|
||||
status:'SYSTEM_WITHHELD',
|
||||
actions: [{
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'LINKS',
|
||||
metadata: {
|
||||
links: comment.body,
|
||||
}
|
||||
}],
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks to see if the user making the comment is allowed to do so
|
||||
// considering their reliability (Trust) status.
|
||||
(context) => {
|
||||
if (context.user && context.user.metadata) {
|
||||
|
||||
// If the user is not a reliable commenter (passed the unreliability
|
||||
// threshold by having too many rejected comments) then we can change the
|
||||
// status of the comment to `SYSTEM_WITHHELD`, therefore pushing the user's
|
||||
// comments away from the public eye until a moderator can manage them. This of
|
||||
// course can only be applied if the comment's current status is `NONE`,
|
||||
// we don't want to interfere if the comment was rejected.
|
||||
if (KarmaService.isReliable('comment', context.user.metadata.trust) === false) {
|
||||
|
||||
// Add the flag related to Trust to the comment.
|
||||
return {
|
||||
status: 'SYSTEM_WITHHELD',
|
||||
actions: [{
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'TRUST',
|
||||
metadata: {
|
||||
trust: context.user.metadata.trust,
|
||||
}
|
||||
}],
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks to see if the comment was already perscribed a status.
|
||||
(context, comment) => {
|
||||
|
||||
// If the status was already defined, don't redefine it. It's only defined
|
||||
// when specific external conditions exist, we don't want to override that.
|
||||
if (comment.status && comment.status.length > 0) {
|
||||
return {
|
||||
status: comment.status,
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
// This phase checks to see if the settings have premod enabled, if they do,
|
||||
// the comment is premod, otherwise, it's just none.
|
||||
(context, comment, {assetSettings: {moderation}}) => {
|
||||
|
||||
// If the settings say that we're in premod mode, then the comment is in
|
||||
// premod status.
|
||||
if (moderation === 'PRE') {
|
||||
return {
|
||||
status: 'PREMOD',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'NONE',
|
||||
};
|
||||
}
|
||||
];
|
||||
|
||||
/**
|
||||
* This resolves a given comment's status to take into account moderator actions
|
||||
* are applied.
|
||||
@@ -233,68 +397,44 @@ const filterNewComment = async (context, {body, asset_id}) => {
|
||||
* @param {Object} [wordlist={}] the results of the wordlist scan
|
||||
* @return {Promise} resolves to the comment's status
|
||||
*/
|
||||
const resolveNewCommentStatus = async (context, {asset_id, body, status}, wordlist = {}, settings = {}) => {
|
||||
let {user} = context;
|
||||
const resolveCommentModeration = async (context, comment) => {
|
||||
|
||||
// Check to see if the body is too short, if it is, then complain about it!
|
||||
if (body.length < 2) {
|
||||
throw errors.ErrCommentTooShort;
|
||||
}
|
||||
// First we filter the comment contents to ensure that we note any validation
|
||||
// issues.
|
||||
let [wordlist, settings] = await filterNewComment(context, comment);
|
||||
|
||||
// If the status was already defined, don't redefine it. It's only defined
|
||||
// when specific external conditions exist, we don't want to override that.
|
||||
if (status && status.length > 0) {
|
||||
return status;
|
||||
}
|
||||
|
||||
// Decide the status based on whether or not the current asset/settings
|
||||
// has pre-mod enabled or not. If the comment was rejected based on the
|
||||
// wordlist, then reject it, otherwise if the moderation setting is
|
||||
// premod, set it to `premod`.
|
||||
if (wordlist.banned) {
|
||||
return 'REJECTED';
|
||||
}
|
||||
|
||||
if (settings.premodLinksEnable && linkify.test(body)) {
|
||||
return 'SYSTEM_WITHHELD';
|
||||
}
|
||||
|
||||
let asset = await AssetsService.findById(asset_id);
|
||||
// Get the asset from the loader.
|
||||
const asset = await context.loaders.Assets.getByID.load(comment.asset_id);
|
||||
if (!asset) {
|
||||
|
||||
// And leave now if this asset wasn't found.
|
||||
throw errors.ErrNotFound;
|
||||
}
|
||||
|
||||
// Check to see if the asset has closed commenting...
|
||||
if (asset.isClosed) {
|
||||
throw new errors.ErrAssetCommentingClosed(asset.closedMessage);
|
||||
}
|
||||
// Combine the asset and the settings to get the asset settings.
|
||||
const assetSettings = await AssetsService.rectifySettings(asset, settings);
|
||||
|
||||
// Return `premod` if pre-moderation is enabled and an empty "new" status
|
||||
// in the event that it is not in pre-moderation mode.
|
||||
let {moderation, charCountEnable, charCount} = await AssetsService.rectifySettings(asset, settings);
|
||||
// Loop over all the moderation phases and see if we've resolved the status.
|
||||
for (const phase of moderationPhases) {
|
||||
const result = await phase(context, comment, {
|
||||
asset,
|
||||
assetSettings,
|
||||
settings,
|
||||
wordlist,
|
||||
});
|
||||
|
||||
// Reject if the comment is too long
|
||||
if (charCountEnable && body.length > charCount) {
|
||||
return 'REJECTED';
|
||||
}
|
||||
if (result) {
|
||||
|
||||
if (user && user.metadata) {
|
||||
// Merge the comment and the result together.
|
||||
comment = merge(comment, result);
|
||||
|
||||
// If the user is not a reliable commenter (passed the unreliability
|
||||
// threshold by having too many rejected comments) then we can change the
|
||||
// status of the comment to `SYSTEM_WITHHELD`, therefore pushing the user's
|
||||
// comments away from the public eye until a moderator can manage them. This of
|
||||
// course can only be applied if the comment's current status is `NONE`,
|
||||
// we don't want to interfere if the comment was rejected.
|
||||
if (KarmaService.isReliable('comment', user.metadata.trust) === false) {
|
||||
|
||||
// Update the response from the comment creation to add the PREMOD so that
|
||||
// that user's UI will reflect the fact that their comment is in pre-mod.
|
||||
return 'SYSTEM_WITHHELD';
|
||||
// If this result contained a status, then we've finished resolving
|
||||
// phases!
|
||||
if (result.status) {
|
||||
return comment.actions;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return moderation === 'PRE' ? 'PREMOD' : 'NONE';
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -305,47 +445,31 @@ const resolveNewCommentStatus = async (context, {asset_id, body, status}, wordli
|
||||
* @param {Object} commentInput the new comment to be created
|
||||
* @return {Promise} resolves to a new comment
|
||||
*/
|
||||
const createPublicComment = async (context, commentInput) => {
|
||||
|
||||
// First we filter the comment contents to ensure that we note any validation
|
||||
// issues.
|
||||
let [wordlist, settings] = await filterNewComment(context, commentInput);
|
||||
const createPublicComment = async (context, comment) => {
|
||||
|
||||
// We then take the wordlist and the comment into consideration when
|
||||
// considering what status to assign the new comment, and resolve the new
|
||||
// status to set the comment to.
|
||||
let status = await resolveNewCommentStatus(context, commentInput, wordlist, settings);
|
||||
let actions = await resolveCommentModeration(context, comment);
|
||||
|
||||
// Then we actually create the comment with the new status.
|
||||
let comment = await createComment(context, commentInput, status);
|
||||
comment = await createComment(context, comment);
|
||||
|
||||
// If the comment has a suspect word or a link, we need to add a
|
||||
// flag to it to indicate that it needs to be looked at.
|
||||
// Otherwise just return the new comment.
|
||||
|
||||
// TODO: Check why the wordlist is undefined
|
||||
|
||||
// If the wordlist has matched the suspect word filter and we haven't disabled
|
||||
// auto-flagging suspect words, then we should flag the comment!
|
||||
if (wordlist != null && wordlist.suspect != null && !DISABLE_AUTOFLAG_SUSPECT_WORDS) {
|
||||
|
||||
// TODO: this is kind of fragile, we should refactor this to resolve
|
||||
// all these const's that we're using like 'COMMENTS', 'FLAG' to be
|
||||
// defined in a checkable schema.
|
||||
await ActionsService.create({
|
||||
item_id: comment.id,
|
||||
item_type: 'COMMENTS',
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'Matched suspect word filter',
|
||||
metadata: {}
|
||||
});
|
||||
}
|
||||
// Create all the actions that were determined during the moderation check
|
||||
// phase.
|
||||
await createActions(comment.id, actions);
|
||||
|
||||
// Finally, we return the comment.
|
||||
return comment;
|
||||
};
|
||||
|
||||
// createActions will for each of the provided actions, create the given action
|
||||
// on the comment at the same time using Promise.all.
|
||||
const createActions = async (item_id, actions = []) => Promise.all(actions.map((action) => merge(action, {
|
||||
item_id,
|
||||
item_type: 'COMMENTS',
|
||||
})).map((action) => ActionsService.create(action)));
|
||||
|
||||
/**
|
||||
* Sets the status of a comment
|
||||
* @param {Object} context graphql context
|
||||
@@ -382,14 +506,19 @@ const setStatus = async ({user, loaders: {Comments}}, {id, status}) => {
|
||||
*/
|
||||
const edit = async (context, {id, asset_id, edit: {body}}) => {
|
||||
|
||||
// Get the wordlist and the settings object.
|
||||
const [wordlist, settings] = await filterNewComment(context, {asset_id, body});
|
||||
// Build up the new comment we're setting. We need to check this with
|
||||
// moderation now.
|
||||
let comment = {id, asset_id, body};
|
||||
|
||||
// Determine the new status of the comment.
|
||||
const status = await resolveNewCommentStatus(context, {asset_id, body}, wordlist, settings);
|
||||
const actions = await resolveCommentModeration(context, comment);
|
||||
|
||||
// Execute the edit.
|
||||
const comment = await CommentsService.edit({id, author_id: context.user.id, body, status});
|
||||
comment = await CommentsService.edit({id, author_id: context.user.id, body, status: comment.status});
|
||||
|
||||
// Create all the actions that were determined during the moderation check
|
||||
// phase.
|
||||
await createActions(comment.id, actions);
|
||||
|
||||
// Publish the edited comment via the subscription.
|
||||
context.pubsub.publish('commentEdited', comment);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
const {getScores, isToxic} = require('./perspective');
|
||||
const {ErrToxic} = require('./errors');
|
||||
const ActionsService = require('../../../services/actions');
|
||||
|
||||
// We don't add the hooks during _test_ as the perspective API is not available.
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
@@ -12,53 +11,36 @@ module.exports = {
|
||||
createComment: {
|
||||
async pre(_, {input}, _context, _info) {
|
||||
|
||||
let scores;
|
||||
|
||||
// Try getting scores.
|
||||
let scores;
|
||||
try {
|
||||
scores = await getScores(input.body);
|
||||
}
|
||||
catch(err) {
|
||||
} catch(err) {
|
||||
|
||||
// Warn and let mutation pass.
|
||||
console.trace(err);
|
||||
return;
|
||||
}
|
||||
|
||||
const commentIsToxic = isToxic(scores);
|
||||
|
||||
if (input.checkToxicity && commentIsToxic) {
|
||||
throw ErrToxic;
|
||||
}
|
||||
|
||||
// attach scores to metadata.
|
||||
// Attach scores to metadata.
|
||||
input.metadata = Object.assign({}, input.metadata, {
|
||||
perspective: scores,
|
||||
});
|
||||
|
||||
if (commentIsToxic) {
|
||||
if (isToxic(scores)) {
|
||||
if (input.checkToxicity) {
|
||||
throw ErrToxic;
|
||||
}
|
||||
|
||||
input.status = 'SYSTEM_WITHHELD';
|
||||
}
|
||||
},
|
||||
async post(_, _input, _context, _info, result) {
|
||||
const metadata = result.comment.metadata;
|
||||
if (metadata.perspective && isToxic(metadata.perspective)) {
|
||||
|
||||
// TODO: this is kind of fragile, we should refactor this to resolve
|
||||
// all these const's that we're using like 'COMMENTS', 'FLAG' to be
|
||||
// defined in a checkable schema.
|
||||
|
||||
// Add a flag to the comment.
|
||||
await ActionsService.create({
|
||||
item_id: result.comment.id,
|
||||
item_type: 'COMMENTS',
|
||||
input.actions = input.actions && input.actions.length >= 0 ? input.actions : [];
|
||||
input.actions.push({
|
||||
action_type: 'FLAG',
|
||||
user_id: null,
|
||||
group_id: 'Comment contains toxic language',
|
||||
metadata: {}
|
||||
});
|
||||
}
|
||||
return result;
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -118,6 +118,8 @@ class Wordlist {
|
||||
// word (suspect).
|
||||
return errors;
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -167,7 +167,7 @@ describe('graph.mutations.createComment', () => {
|
||||
|
||||
[
|
||||
{message: 'comment does not contain banned/suspect words', body: 'This is such a nice comment!', status: 'NONE', flagged: false},
|
||||
{message: 'comment contains banned words', body: 'This is the WORST comment!', status: 'REJECTED', flagged: false},
|
||||
{message: 'comment contains banned words', body: 'This is the WORST comment!', status: 'REJECTED', flagged: true},
|
||||
{message: 'comment contains suspect words', body: 'This is the EH comment!', status: 'NONE', flagged: true}
|
||||
].forEach(({message, body, status, flagged}) => {
|
||||
describe(message, () => {
|
||||
@@ -222,6 +222,9 @@ describe('graph.mutations.createComment', () => {
|
||||
|
||||
return graphql(schema, query, {}, context)
|
||||
.then(({data, errors}) => {
|
||||
if (errors) {
|
||||
console.error(errors);
|
||||
}
|
||||
expect(errors).to.be.undefined;
|
||||
expect(data.createComment).to.have.property('comment').not.null;
|
||||
expect(data.createComment).to.have.property('errors').null;
|
||||
|
||||
@@ -89,7 +89,7 @@ describe('services.Wordlist', () => {
|
||||
'I have bad $ hit lling',
|
||||
'That\'s a p***ch!',
|
||||
].forEach((word) => {
|
||||
expect(wordlist.scan('body', word)).to.be.undefined;
|
||||
expect(wordlist.scan('body', word)).to.be.deep.equal({});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user