diff --git a/plugins/talk-plugin-toxic-comments/client/components/CheckToxicityHook.js b/plugins/talk-plugin-toxic-comments/client/components/CheckToxicityHook.js index 2e4c66d3e..22c378435 100644 --- a/plugins/talk-plugin-toxic-comments/client/components/CheckToxicityHook.js +++ b/plugins/talk-plugin-toxic-comments/client/components/CheckToxicityHook.js @@ -1,10 +1,19 @@ import React from 'react'; +/** + * CheckToxicityHook adds hooks to the `commentBox` + * that handles checking a comment for toxicity. + */ export default class CheckToxicityHook extends React.Component { + + // checked signifies if we already sent a request with the `checkToxicity` set to true. checked = false; componentDidMount() { this.toxicityPreHook = this.props.registerHook('preSubmit', (input) => { + + // If we haven't check the toxicity yet, make sure to include `checkToxicity=true` in the mutation. + // Otherwise post comment without checking the toxicity. if (!this.checked) { input.checkToxicity = true; this.checked = true; @@ -12,6 +21,8 @@ export default class CheckToxicityHook extends React.Component { }); this.toxicityPostHook = this.props.registerHook('postSubmit', () => { + + // Reset `checked` after comment was successfully posted. this.checked = false; }); } diff --git a/plugins/talk-plugin-toxic-comments/client/index.js b/plugins/talk-plugin-toxic-comments/client/index.js index 46f8c0fc0..c364a2a00 100644 --- a/plugins/talk-plugin-toxic-comments/client/index.js +++ b/plugins/talk-plugin-toxic-comments/client/index.js @@ -1,11 +1,6 @@ import translations from './translations.yml'; import CheckToxicityHook from './components/CheckToxicityHook'; -/** - * coral-plugin-offtopic depends on coral-plugin-viewing-options - * in other to display filter and use the streamViewingOptions slot - */ - export default { translations, slots: { diff --git a/plugins/talk-plugin-toxic-comments/server/apiKey.js b/plugins/talk-plugin-toxic-comments/server/apiKey.js deleted file mode 100644 index 43e02d154..000000000 --- a/plugins/talk-plugin-toxic-comments/server/apiKey.js +++ /dev/null @@ -1,6 +0,0 @@ -const apiKey = process.env.TALK_PERSPECTIVE_API_KEY; -if(!apiKey) { - throw new Error('Please set the TALK_PERSPECTIVE_API_KEY environment variable to use the toxic-comments plugin. Visit https://www.perspectiveapi.com/ to request API access.'); -} - -module.exports = apiKey; diff --git a/plugins/talk-plugin-toxic-comments/server/config.js b/plugins/talk-plugin-toxic-comments/server/config.js new file mode 100644 index 000000000..67155cd45 --- /dev/null +++ b/plugins/talk-plugin-toxic-comments/server/config.js @@ -0,0 +1,5 @@ +module.exports = { + API_ENDPOINT: 'https://commentanalyzer.googleapis.com/v1alpha1', + API_KEY: process.env.NODE_ENV !== 'test' ? process.env.TALK_PERSPECTIVE_API_KEY : '', + TOXICITY_THRESHOLD: process.env.TALK_TOXICITY_THRESHOLD || 0.8, +}; diff --git a/plugins/talk-plugin-toxic-comments/server/constants.js b/plugins/talk-plugin-toxic-comments/server/constants.js deleted file mode 100644 index 1aee300a6..000000000 --- a/plugins/talk-plugin-toxic-comments/server/constants.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - TOXICITY_THRESHOLD: 0.85, -}; diff --git a/plugins/talk-plugin-toxic-comments/server/errors.js b/plugins/talk-plugin-toxic-comments/server/errors.js index 17a2d20c6..e5c77ff78 100644 --- a/plugins/talk-plugin-toxic-comments/server/errors.js +++ b/plugins/talk-plugin-toxic-comments/server/errors.js @@ -1,5 +1,8 @@ -const {APIError} = require('../../../errors'); +const {APIError} = require('errors'); +// ErrToxic is sent during a `CreateComment` mutation where +// `input.checkToxicity` is set to true and the comment contains +// toxic language as determined by the perspective service. const ErrToxic = new APIError('Comment is toxic', { status: 400, translation_key: 'COMMENT_IS_TOXIC', diff --git a/plugins/talk-plugin-toxic-comments/server/hooks.js b/plugins/talk-plugin-toxic-comments/server/hooks.js index 49f7f5d1c..32f61b905 100644 --- a/plugins/talk-plugin-toxic-comments/server/hooks.js +++ b/plugins/talk-plugin-toxic-comments/server/hooks.js @@ -1,49 +1,44 @@ -const perspective = require('./perspective'); +const {getScores, isToxic} = require('./perspective'); const {ErrToxic} = require('./errors'); -const {TOXICITY_THRESHOLD} = require('./constants'); const ActionsService = require('../../../services/actions'); +// We don't add the hooks during _test_ as the perspective API is not available. +if (process.env.NODE_ENV === 'test') { + return null; +} + module.exports = { RootMutation: { createComment: { async pre(_, {input}, _context, _info) { - // Don't call out to perspective when running tests. - if (process.env.NODE_ENV === 'test') { - return; - } - - const apiKey = require('./apiKey'); - // TODO: handle timeouts. - const scores = await perspective.getScores(apiKey, input.body); + const scores = await getScores(input.body); + const commentIsToxic = isToxic(scores); - const isToxic = scores.SEVERE_TOXICITY.summaryScore > TOXICITY_THRESHOLD; - if (input.checkToxicity && isToxic) { + if (input.checkToxicity && commentIsToxic) { throw ErrToxic; } + + // attach scores to metadata. input.metadata = Object.assign({}, input.metadata, { perspective: scores, }); - if (isToxic) { + if (commentIsToxic) { + + // TODO: this should have a different status than Premod. input.status = 'PREMOD'; } }, async post(_, _input, _context, _info, result) { - - // Perspective is not available when running tests. - if (process.env.NODE_ENV === 'test') { - return result; - } - - const score = result.comment.metadata.perspective.SEVERE_TOXICITY.summaryScore; - const isToxic = score > TOXICITY_THRESHOLD; - if (isToxic) { + if (isToxic(result.comment.metadata.perspective)) { // TODO: this is kind of fragile, we should refactor this to resolve // all these const's that we're using like 'COMMENTS', 'FLAG' to be // defined in a checkable schema. + + // Add a flag to the comment. await ActionsService.create({ item_id: result.comment.id, item_type: 'COMMENTS', diff --git a/plugins/talk-plugin-toxic-comments/server/perspective.js b/plugins/talk-plugin-toxic-comments/server/perspective.js index e863b4777..e66e6ed10 100644 --- a/plugins/talk-plugin-toxic-comments/server/perspective.js +++ b/plugins/talk-plugin-toxic-comments/server/perspective.js @@ -1,9 +1,13 @@ const fetch = require('node-fetch'); +const {API_ENDPOINT, API_KEY, TOXICITY_THRESHOLD} = require ('./config'); -const API_ENPOINT = 'https://commentanalyzer.googleapis.com/v1alpha1'; - -async function getScores(apiKey, text) { - const response = await fetch(`${API_ENPOINT}/comments:analyze?key=${apiKey}`, { +/** + * Get scores from the perspective api + * @param {string} text to be anaylized + * @return {object} object containing toxicity scores + */ +async function getScores(text) { + const response = await fetch(`${API_ENDPOINT}/comments:analyze?key=${API_KEY}`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -32,6 +36,29 @@ async function getScores(apiKey, text) { }; } +/** + * Get toxicity probability + * @param {object} scores as returned by `getScores` + * @return {number} toxicity probability from 0 - 1.0 + */ +function getProbability(scores) { + return scores.SEVERE_TOXICITY.summaryScore; +} + +/** + * isToxics determines if given probabilty or scores meets the toxicity threshold. + * @param {object|number} scores or probability + * @return {boolean} + */ +function isToxic(scoresOrProbability) { + const probability = typeof scoresOrProbability === 'object' + ? getProbability(scoresOrProbability) + : scoresOrProbability; + return probability > TOXICITY_THRESHOLD; +} + module.exports = { getScores, + getProbability, + isToxic, }; diff --git a/plugins/talk-plugin-toxic-comments/server/typeDefs.graphql b/plugins/talk-plugin-toxic-comments/server/typeDefs.graphql index 7352db00f..b4dfc8345 100644 --- a/plugins/talk-plugin-toxic-comments/server/typeDefs.graphql +++ b/plugins/talk-plugin-toxic-comments/server/typeDefs.graphql @@ -1,4 +1,7 @@ input CreateCommentInput { + + # If true, the mutation will fail when the + # body contains toxic language. checkToxicity: Boolean }