Put toxic comment into premod and add flag

graph/mutators/comment.js

@@ -215,7 +215,7 @@ const filterNewComment = (context, {body, asset_id}) => {
  * @param  {Object} [wordlist={}] the results of the wordlist scan
  * @return {Promise}              resolves to the comment's status
  */
-const resolveNewCommentStatus = async (context, {asset_id, body}, wordlist = {}, settings = {}) => {
+const resolveNewCommentStatus = async (context, {asset_id, body, status}, wordlist = {}, settings = {}) => {
   let {user} = context;
 
   // Check to see if the body is too short, if it is, then complain about it!
@@ -270,7 +270,7 @@ const resolveNewCommentStatus = async (context, {asset_id, body}, wordlist = {},
     }
   }
 
-  return moderation === 'PRE' ? 'PREMOD' : 'NONE';
+  return (moderation === 'PRE' || status === 'PREMOD') ? 'PREMOD' : 'NONE';
 };
 
 /**

plugins/talk-plugin-toxic-comments/server/constants.js

@@ -1,3 +1,3 @@
 module.exports = {
-  TOXICITY_THRESHOLD: 0.8,
+  TOXICITY_THRESHOLD: 0.85,
 };

plugins/talk-plugin-toxic-comments/server/hooks.js

@@ -1,6 +1,7 @@
 const perspective = require('./perspective');
 const {ErrToxic} = require('./errors');
 const {TOXICITY_THRESHOLD} = require('./constants');
+const ActionsService = require('../../../services/actions');
 
 module.exports = {
   RootMutation: {
@@ -13,7 +14,10 @@ module.exports = {
         }
 
         const apiKey = require('./apiKey');
+
+        // TODO: handle timeouts.
         const scores = await perspective.getScores(apiKey, input.body);
+
         const isToxic = scores.SEVERE_TOXICITY.summaryScore > TOXICITY_THRESHOLD;
         if (input.checkToxicity && isToxic) {
           throw ErrToxic;
@@ -23,10 +27,34 @@ module.exports = {
         });
 
         if (isToxic) {
-
-          // TODO: Flag comment as toxic and put in ?premod?.
+          input.status = 'PREMOD';
         }
       },
+      async post(_, _input, _context, _info, result) {
+
+        // Perspective is not available when running tests.
+        if (process.env.NODE_ENV === 'test') {
+          return result;
+        }
+
+        const score = result.comment.metadata.perspective.SEVERE_TOXICITY.summaryScore;
+        const isToxic = score > TOXICITY_THRESHOLD;
+        if (isToxic) {
+
+          // TODO: this is kind of fragile, we should refactor this to resolve
+          // all these const's that we're using like 'COMMENTS', 'FLAG' to be
+          // defined in a checkable schema.
+          await ActionsService.create({
+            item_id: result.comment.id,
+            item_type: 'COMMENTS',
+            action_type: 'FLAG',
+            user_id: null,
+            group_id: 'Comment contains toxic language',
+            metadata: {}
+          });
+        }
+        return result;
+      },
     },
   },
 };