Put toxic comment into premod and add flag

This commit is contained in:
Chi Vinh Le
2017-09-06 22:58:52 +07:00
parent b3fe3afd4c
commit 9b0caa42cd
3 changed files with 33 additions and 5 deletions
@@ -1,3 +1,3 @@
module.exports = {
TOXICITY_THRESHOLD: 0.8,
TOXICITY_THRESHOLD: 0.85,
};
@@ -1,6 +1,7 @@
const perspective = require('./perspective');
const {ErrToxic} = require('./errors');
const {TOXICITY_THRESHOLD} = require('./constants');
const ActionsService = require('../../../services/actions');
module.exports = {
RootMutation: {
@@ -13,7 +14,10 @@ module.exports = {
}
const apiKey = require('./apiKey');
// TODO: handle timeouts.
const scores = await perspective.getScores(apiKey, input.body);
const isToxic = scores.SEVERE_TOXICITY.summaryScore > TOXICITY_THRESHOLD;
if (input.checkToxicity && isToxic) {
throw ErrToxic;
@@ -23,10 +27,34 @@ module.exports = {
});
if (isToxic) {
// TODO: Flag comment as toxic and put in ?premod?.
input.status = 'PREMOD';
}
},
async post(_, _input, _context, _info, result) {
// Perspective is not available when running tests.
if (process.env.NODE_ENV === 'test') {
return result;
}
const score = result.comment.metadata.perspective.SEVERE_TOXICITY.summaryScore;
const isToxic = score > TOXICITY_THRESHOLD;
if (isToxic) {
// TODO: this is kind of fragile, we should refactor this to resolve
// all these const's that we're using like 'COMMENTS', 'FLAG' to be
// defined in a checkable schema.
await ActionsService.create({
item_id: result.comment.id,
item_type: 'COMMENTS',
action_type: 'FLAG',
user_id: null,
group_id: 'Comment contains toxic language',
metadata: {}
});
}
return result;
},
},
},
};