From 51bfde8cf8f73a6b23b647335a6f8f67b3459328 Mon Sep 17 00:00:00 2001
From: Wyatt Johnson <wyattjoh@gmail.com>
Date: Thu, 5 Dec 2019 18:17:07 +0000
Subject: [PATCH] feat: added toxicity multi-language support (#2737)

---
 CONTRIBUTING.md                               |  5 ++
 src/core/common/helpers/i18n/locales.ts       |  2 +-
 .../comments/pipeline/phases/toxic.ts         | 48 ++++++++++++++++---
 3 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 87a7bbcb2..fc30b7cd2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -64,3 +64,8 @@ where the string is used.
 
 Once a language has enough coverage, it should be added to
 `src/core/common/helpers/i18n/locales.ts`.
+
+The [Perspective API](https://github.com/conversationai/perspectiveapi/blob/master/2-api/methods.md#analyzecomment-request)
+also supports comments in specific languages. When the language is supported in
+Coral and supported by the Perspective API, the language should be added to the
+language map in `src/core/server/services/comments/pipeline/phases/toxic.ts`.
diff --git a/src/core/common/helpers/i18n/locales.ts b/src/core/common/helpers/i18n/locales.ts
index ea8abecb3..0f4ced5be 100644
--- a/src/core/common/helpers/i18n/locales.ts
+++ b/src/core/common/helpers/i18n/locales.ts
@@ -1,6 +1,6 @@
 /**
  * LanguageCode is the type represented by the internally identifiable types for
- * the different languages that can be supported.
+ * the different languages that can be supported in the BCP 47 format.
  */
 export type LanguageCode =
   | "en-US"
diff --git a/src/core/server/services/comments/pipeline/phases/toxic.ts b/src/core/server/services/comments/pipeline/phases/toxic.ts
index e2a5b76e8..e6f750188 100644
--- a/src/core/server/services/comments/pipeline/phases/toxic.ts
+++ b/src/core/server/services/comments/pipeline/phases/toxic.ts
@@ -9,6 +9,7 @@ import {
   TOXICITY_MODEL_DEFAULT,
   TOXICITY_THRESHOLD_DEFAULT,
 } from "coral-common/constants";
+import { LanguageCode } from "coral-common/helpers";
 import { Omit } from "coral-common/types";
 import { ToxicCommentError } from "coral-server/errors";
 import logger from "coral-server/logger";
@@ -16,6 +17,7 @@ import { ACTION_TYPE } from "coral-server/models/action/comment";
 import {
   IntermediateModerationPhase,
   IntermediatePhaseResult,
+  ModerationPhaseContext,
 } from "coral-server/services/comments/pipeline";
 
 import {
@@ -26,11 +28,14 @@ import {
 
 export const toxic: IntermediateModerationPhase = async ({
   tenant,
-  comment,
   nudge,
   log,
-}): Promise<IntermediatePhaseResult | void> => {
-  if (!comment.body) {
+  htmlStripped,
+}: Pick<
+  ModerationPhaseContext,
+  "tenant" | "nudge" | "log" | "htmlStripped"
+>): Promise<IntermediatePhaseResult | void> => {
+  if (!htmlStripped) {
     return;
   }
 
@@ -89,15 +94,21 @@ export const toxic: IntermediateModerationPhase = async ({
     // Pull the custom model out.
     const model = integration.model || TOXICITY_MODEL_DEFAULT;
 
+    // Get the language from the tenant's set language. This won't be a 1-1
+    // mapping because the Perspective API doesn't support all the languages
+    // that Coral supports in production.
+    const language = convertLanguage(tenant.locale);
+
     // Call into the Toxic comment API.
     const score = await getScore(
-      comment.body,
+      htmlStripped,
       {
         endpoint,
         key: integration.key,
         doNotStore,
         model,
       },
+      language,
       timeout
     );
 
@@ -144,6 +155,31 @@ export const toxic: IntermediateModerationPhase = async ({
   }
 };
 
+/**
+ * Language is the language key that is supported by the Perspective API in the
+ * ISO 631-1 format.
+ */
+type PerspectiveLanguage = "en" | "es" | "fr" | "de";
+
+/**
+ * convertLanguage returns the language code for the related Perspective API
+ * model in the ISO 631-1 format.
+ *
+ * @param locale the language on the tenant in the BCP 47 format.
+ */
+function convertLanguage(locale: LanguageCode): PerspectiveLanguage {
+  switch (locale) {
+    case "en-US":
+      return "en";
+    case "es":
+      return "es";
+    case "de":
+      return "de";
+    default:
+      return "en";
+  }
+}
+
 /**
  * getScore will return the toxicity score for the comment text.
  *
@@ -160,6 +196,7 @@ async function getScore(
     model,
     doNotStore,
   }: Required<Omit<GQLPerspectiveExternalIntegration, "enabled" | "threshold">>,
+  language: PerspectiveLanguage,
   timeout: number
 ): Promise<number> {
   // Prepare the URL to send the command to.
@@ -178,8 +215,7 @@ async function getScore(
         comment: {
           text,
         },
-        // TODO: (wyattjoh) support other languages.
-        languages: ["en"],
+        languages: [language],
         doNotStore,
         requestedAttributes: {
           [model]: {},