[CORl-278] Perspective Model (#2337)

* feat: support switching the perspective model

* feat: support changing the model on admin

* fix: linting

* fix: removed defaulted value
This commit is contained in:
Wyatt Johnson
2019-06-28 22:10:32 +00:00
committed by GitHub
parent e72b15c505
commit 76033118e5
12 changed files with 154 additions and 19 deletions
@@ -763,6 +763,11 @@ type PerspectiveExternalIntegration {
"""
threshold: Float
"""
model is the Perspective model to use.
"""
model: String
"""
When True, comments sent will not be stored by the Google Perspective API.
"""
@@ -2693,6 +2698,11 @@ input SettingsPerspectiveExternalIntegrationInput {
"""
key: String
"""
model is the name of the Perspective model to use.
"""
model: String
"""
The threshold that given a specific toxic comment score, the comment will
be marked by Coral as toxic.
@@ -2,6 +2,10 @@ import { isNil } from "lodash";
import ms from "ms";
import fetch from "node-fetch";
import {
TOXICITY_MODEL_DEFAULT,
TOXICITY_THRESHOLD_DEFAULT,
} from "coral-common/constants";
import { Omit } from "coral-common/types";
import { ToxicCommentError } from "coral-server/errors";
import {
@@ -56,8 +60,7 @@ export const toxic: IntermediateModerationPhase = async ({
let threshold = integration.threshold;
if (isNil(threshold)) {
// TODO: (wyattjoh) replace hardcoded default with config.
threshold = 0.8;
threshold = TOXICITY_THRESHOLD_DEFAULT / 100;
log.trace(
{ threshold },
@@ -81,17 +84,17 @@ export const toxic: IntermediateModerationPhase = async ({
try {
logger.trace("checking comment toxicity");
// TODO: (wyattjoh) support custom toxicity model.
const model = "TOXICITY";
// Pull the custom model out.
const model = integration.model || TOXICITY_MODEL_DEFAULT;
// Call into the Toxic comment API.
const score = await getScore(
comment.body,
model,
{
endpoint,
key: integration.key,
doNotStore,
model,
},
timeout
);
@@ -142,10 +145,10 @@ export const toxic: IntermediateModerationPhase = async ({
*/
async function getScore(
text: string,
model: string,
{
key,
endpoint,
model,
doNotStore,
}: Required<Omit<GQLPerspectiveExternalIntegration, "enabled" | "threshold">>,
timeout: number