feat: added toxic labels (#2396)

src/core/server/graph/tenant/resolvers/CommentRevision.ts

@@ -14,5 +14,7 @@ export const CommentRevision: Required<
   comment: w => w.comment,
   actionCounts: w => decodeActionCounts(w.revision.actionCounts),
   body: w => w.revision.body,
+  // Defaults to an empty object if not set on the revision.
+  metadata: w => w.revision.metadata || {},
   createdAt: w => w.revision.createdAt,
 };

src/core/server/graph/tenant/schema/schema.graphql

@@ -750,12 +750,12 @@ type PerspectiveExternalIntegration {
   """
   The endpoint that Coral should use to communicate with the perspective API.
   """
-  endpoint: String
+  endpoint: String @auth(roles: [ADMIN])
 
   """
   The key for the Perspective API integration.
   """
-  key: String
+  key: String @auth(roles: [ADMIN])
 
   """
   The threshold that given a specific toxic comment score, the comment will
@@ -766,12 +766,12 @@ type PerspectiveExternalIntegration {
   """
   model is the Perspective model to use.
   """
-  model: String
+  model: String @auth(roles: [ADMIN])
 
   """
   When True, comments sent will not be stored by the Google Perspective API.
   """
-  doNotStore: Boolean
+  doNotStore: Boolean @auth(roles: [ADMIN])
 }
 
 type ExternalIntegrations {
@@ -1158,7 +1158,7 @@ type Settings {
   """
   integrations contains all the external integrations that can be enabled.
   """
-  integrations: ExternalIntegrations! @auth(roles: [ADMIN])
+  integrations: ExternalIntegrations! @auth(roles: [ADMIN, MODERATOR])
 
   """
   karma is the set of settings related to how user Trust and Karma are
@@ -1705,6 +1705,23 @@ type CommentModerationActionConnection {
   pageInfo: PageInfo!
 }
 
+type CommentRevisionPerspectiveMetadata {
+  """
+  score is the value detected from the perspective API. This is returned as the
+  percentage chance it would be considered toxic and can be compared to the
+  defined threshold value.
+  """
+  score: Float!
+}
+
+type CommentRevisionMetadata {
+  """
+  perspective stores metadata associated with the pipeline analysis of this
+  revision's body.
+  """
+  perspective: CommentRevisionPerspectiveMetadata
+}
+
 type CommentRevision {
   """
   id is the identifier of the CommentRevision.
@@ -1729,6 +1746,11 @@ type CommentRevision {
   """
   body: String
 
+  """
+  metadata stores details on a CommentRevision.
+  """
+  metadata: CommentRevisionMetadata! @auth(roles: [ADMIN, MODERATOR])
+
   """
   createdAt is the time that the CommentRevision was created.
   """

src/core/server/models/comment/index.ts

@@ -42,6 +42,15 @@ function collection<T = Comment>(mongo: Db) {
   return mongo.collection<Readonly<T>>("comments");
 }
 
+export interface RevisionMetadata {
+  akismet?: boolean;
+  linkCount?: number;
+  perspective?: {
+    score: number;
+    model: string;
+  };
+}
+
 /**
  * Revision stores a Comment's body for a specific edit. Actions can be tied to
  * a Revision, as can moderation actions.
@@ -62,6 +71,11 @@ export interface Revision {
    */
   actionCounts: EncodedCommentActionCounts;
 
+  /**
+   * metadata stores properties on this revision.
+   */
+  metadata: RevisionMetadata;
+
   /**
    * createdAt is the date that this revision was created at.
    */
@@ -140,11 +154,6 @@ export interface Comment extends TenantResource {
    */
   childCount: number;
 
-  /**
-   * metadata stores the deep Comment properties.
-   */
-  metadata?: Record<string, any>;
-
   /**
    * createdAt is the date that this Comment was created.
    */
@@ -240,6 +249,7 @@ export type CreateCommentInput = Omit<
   | "deletedAt"
 > &
   Required<Pick<Revision, "body">> &
+  Pick<Revision, "metadata"> &
   Partial<Pick<Comment, "actionCounts">>;
 
 export async function createComment(
@@ -249,13 +259,14 @@ export async function createComment(
   now = new Date()
 ) {
   // Pull out some useful properties from the input.
-  const { body, actionCounts = {}, ...rest } = input;
+  const { body, actionCounts = {}, metadata, ...rest } = input;
 
   // Generate the revision.
   const revision: Revision = {
     id: uuid.v4(),
     body,
     actionCounts,
+    metadata,
     createdAt: now,
   };
 
@@ -311,17 +322,14 @@ export async function pushChildCommentIDOntoParent(
   return result.value;
 }
 
-export type EditCommentInput = Pick<
-  Comment,
-  "id" | "authorID" | "status" | "metadata"
-> & {
+export type EditCommentInput = Pick<Comment, "id" | "authorID" | "status"> & {
   /**
    * lastEditableCommentCreatedAt is the date that the last comment would have
    * been editable. It is generally derived from the tenant's
    * `editCommentWindowLength` property.
    */
   lastEditableCommentCreatedAt: Date;
-} & Required<Pick<Revision, "body">> &
+} & Required<Pick<Revision, "body" | "metadata">> &
   Partial<Pick<Comment, "actionCounts">>;
 
 // Only comments with the following status's can be edited.
@@ -401,17 +409,12 @@ export async function editComment(
     id: uuid.v4(),
     body,
     actionCounts,
+    metadata,
     createdAt: now,
   };
 
   const update: Record<string, any> = {
-    $set: {
-      status,
-      // Embed all the metadata properties, this may override the existing
-      // metadata, but we won't replace metadata that has been recalculated.
-      // TODO: (wyattjoh) consider if we want to replace the metadata for edited comments instead of supplementing it
-      ...dotize({ metadata }),
-    },
+    $set: { status },
     $push: {
       revisions: revision,
     },

src/core/server/services/comments/index.ts

@@ -248,7 +248,7 @@ export async function create(
 
 export type EditComment = Omit<
   EditCommentInput,
-  "status" | "authorID" | "lastEditableCommentCreatedAt"
+  "status" | "authorID" | "lastEditableCommentCreatedAt" | "metadata"
 >;
 
 export async function edit(

src/core/server/services/comments/pipeline/index.spec.ts

@@ -40,15 +40,15 @@ describe("compose", () => {
   it("merges the metadata", async () => {
     const status = GQLCOMMENT_STATUS.APPROVED;
     const enhanced = compose([
-      () => ({ metadata: { first: true } }),
-      () => ({ status, metadata: { second: true } }),
-      () => ({ metadata: { third: true } }),
+      () => ({ metadata: { akismet: true } }),
+      () => ({ metadata: { linkCount: 1 } }),
+      () => ({ status, metadata: { akismet: false } }),
     ]);
 
     await expect(enhanced(context)).resolves.toEqual({
       body: context.comment.body,
       status,
-      metadata: { first: true, second: true },
+      metadata: { akismet: false, linkCount: 1 },
       actions: [],
       tags: [],
     });
@@ -104,14 +104,14 @@ describe("compose", () => {
 
   it("handles when it does not return a status", async () => {
     const enhanced = compose([
-      () => ({ metadata: { first: true } }),
-      () => ({ metadata: { second: true } }),
+      () => ({ metadata: { akismet: true } }),
+      () => ({ metadata: { akismet: false } }),
     ]);
 
     await expect(enhanced(context)).resolves.toEqual({
       body: context.comment.body,
       status: GQLCOMMENT_STATUS.NONE,
-      metadata: { first: true, second: true },
+      metadata: { akismet: false },
       actions: [],
       tags: [],
     });

src/core/server/services/comments/pipeline/index.ts

@@ -1,7 +1,10 @@
 import { Omit, Promiseable, RequireProperty } from "coral-common/types";
 import { GQLCOMMENT_STATUS } from "coral-server/graph/tenant/schema/__generated__/types";
 import { CreateActionInput } from "coral-server/models/action/comment";
-import { EditCommentInput } from "coral-server/models/comment";
+import {
+  EditCommentInput,
+  RevisionMetadata,
+} from "coral-server/models/comment";
 import { CommentTag } from "coral-server/models/comment/tag";
 import { Story } from "coral-server/models/story";
 import { Tenant } from "coral-server/models/tenant";
@@ -18,7 +21,7 @@ export type ModerationAction = Omit<
 export interface PhaseResult {
   actions: ModerationAction[];
   status: GQLCOMMENT_STATUS;
-  metadata: Record<string, any>;
+  metadata: RevisionMetadata;
   body: string;
   tags: CommentTag[];
 }
@@ -33,14 +36,19 @@ export interface ModerationPhaseContext {
   req?: Request;
 }
 
-export type ModerationPhase = (
+export type RootModerationPhase = (
   context: ModerationPhaseContext
 ) => Promiseable<PhaseResult>;
 
 export type IntermediatePhaseResult = Partial<PhaseResult> | void;
 
+export interface IntermediateModerationPhaseContext
+  extends ModerationPhaseContext {
+  metadata: RevisionMetadata;
+}
+
 export type IntermediateModerationPhase = (
-  context: ModerationPhaseContext
+  context: IntermediateModerationPhaseContext
 ) => Promiseable<IntermediatePhaseResult>;
 
 /**
@@ -49,7 +57,7 @@ export type IntermediateModerationPhase = (
  */
 export const compose = (
   phases: IntermediateModerationPhase[]
-): ModerationPhase => async context => {
+): RootModerationPhase => async context => {
   const final: PhaseResult = {
     status: GQLCOMMENT_STATUS.NONE,
     body: context.comment.body,
@@ -65,8 +73,8 @@ export const compose = (
       comment: {
         ...context.comment,
         body: final.body,
-        metadata: final.metadata,
       },
+      metadata: final.metadata,
     });
     if (result) {
       // If this result contained actions, then we should push it into the
@@ -119,4 +127,6 @@ export const compose = (
 /**
  * process the comment and return moderation details.
  */
-export const processForModeration: ModerationPhase = compose(moderationPhases);
+export const processForModeration: RootModerationPhase = compose(
+  moderationPhases
+);

src/core/server/services/comments/pipeline/phases/commentLength.ts

@@ -27,7 +27,6 @@ const testCharCount = (settings: Partial<Settings>, length: number) => {
 };
 
 export const commentLength: IntermediateModerationPhase = ({
-  story,
   tenant,
   comment,
 }): IntermediatePhaseResult | void => {

src/core/server/services/comments/pipeline/phases/detectLinks.ts

@@ -4,7 +4,7 @@ import {
   GQLCOMMENT_STATUS,
 } from "coral-server/graph/tenant/schema/__generated__/types";
 import { ACTION_TYPE } from "coral-server/models/action/comment";
-import { Comment } from "coral-server/models/comment";
+import { RevisionMetadata } from "coral-server/models/comment";
 import { GlobalModerationSettings } from "coral-server/models/settings";
 import {
   IntermediateModerationPhase,
@@ -13,21 +13,19 @@ import {
 
 const testPremodLinksEnable = (
   settings: DeepPartial<GlobalModerationSettings>,
-  comment: Pick<Comment, "metadata">
-) =>
-  settings.premodLinksEnable && comment.metadata && comment.metadata.linkCount;
+  metadata: RevisionMetadata
+) => settings.premodLinksEnable && metadata && metadata.linkCount;
 
 // This phase checks the comment if it has any links in it if the check is
 // enabled.
 export const detectLinks: IntermediateModerationPhase = ({
   story,
   tenant,
-  comment,
+  metadata,
 }): IntermediatePhaseResult | void => {
   if (
-    comment &&
-    (testPremodLinksEnable(tenant, comment) ||
-      (story.settings && testPremodLinksEnable(story.settings, comment)))
+    testPremodLinksEnable(tenant, metadata) ||
+    (story.settings && testPremodLinksEnable(story.settings, metadata))
   ) {
     // Add the flag related to Trust to the comment.
     return {

src/core/server/services/comments/pipeline/phases/preModerate.ts

@@ -21,7 +21,6 @@ export const preModerate: IntermediateModerationPhase = ({
   // If the settings say that we're in premod mode, then the comment is in
   // premod status.
 
-  // TODO: (wyattjoh) pull from the story settings.
   if (
     testModerationMode(tenant) ||
     (story.settings && testModerationMode(story.settings))

src/core/server/services/comments/pipeline/phases/spam.ts

@@ -115,7 +115,7 @@ export const spam: IntermediateModerationPhase = async ({
         ],
         metadata: {
           // Store the spam result from Akismet in the Comment metadata.
-          akismet: spam,
+          akismet: isSpam,
         },
       };
     }

src/core/server/services/comments/pipeline/phases/storyClosed.spec.ts

@@ -9,8 +9,6 @@ describe("storyClosed", () => {
       storyClosed({
         story: { closedAt: new Date() } as ModerationPhaseContext["story"],
         tenant: {} as ModerationPhaseContext["tenant"],
-        comment: {} as ModerationPhaseContext["comment"],
-        author: {} as ModerationPhaseContext["author"],
         now: new Date(),
       })
     ).toThrow();
@@ -21,8 +19,6 @@ describe("storyClosed", () => {
         closeCommenting: { auto: true },
       } as ModerationPhaseContext["tenant"],
       now: new Date(),
-      comment: {} as ModerationPhaseContext["comment"],
-      author: {} as ModerationPhaseContext["author"],
     });
 
     expect(() =>
@@ -35,8 +31,6 @@ describe("storyClosed", () => {
           },
         } as ModerationPhaseContext["tenant"],
         now: new Date(),
-        comment: {} as ModerationPhaseContext["comment"],
-        author: {} as ModerationPhaseContext["author"],
       })
     ).toThrow();
   });
@@ -54,8 +48,6 @@ describe("storyClosed", () => {
         tenant: {
           closeCommenting: { auto: true },
         } as ModerationPhaseContext["tenant"],
-        comment: {} as ModerationPhaseContext["comment"],
-        author: {} as ModerationPhaseContext["author"],
         now,
       })
     ).toBeUndefined();
@@ -66,8 +58,6 @@ describe("storyClosed", () => {
         tenant: {
           closeCommenting: { auto: true },
         } as ModerationPhaseContext["tenant"],
-        comment: {} as ModerationPhaseContext["comment"],
-        author: {} as ModerationPhaseContext["author"],
         now,
       })
     ).toBeUndefined();

src/core/server/services/comments/pipeline/phases/storyClosed.ts

@@ -1,16 +1,19 @@
 import { StoryClosedError } from "coral-server/errors";
 import {
-  IntermediateModerationPhase,
   IntermediatePhaseResult,
+  ModerationPhaseContext,
 } from "coral-server/services/comments/pipeline";
 import { getStoryClosedAt } from "coral-server/services/stories";
 
 // This phase checks to see if the story being processed is closed or not.
-export const storyClosed: IntermediateModerationPhase = ({
+export const storyClosed = ({
   story,
   tenant,
   now,
-}): IntermediatePhaseResult | void => {
+}: Pick<
+  ModerationPhaseContext,
+  "story" | "tenant" | "now"
+>): IntermediatePhaseResult | void => {
   const closedAt = getStoryClosedAt(tenant, story);
   if (closedAt && closedAt <= now) {
     throw new StoryClosedError();

src/core/server/services/comments/pipeline/phases/toxic.ts

@@ -3,6 +3,7 @@ import ms from "ms";
 import fetch from "node-fetch";
 
 import {
+  TOXICITY_ENDPOINT_DEFAULT,
   TOXICITY_MODEL_DEFAULT,
   TOXICITY_THRESHOLD_DEFAULT,
 } from "coral-common/constants";
@@ -49,8 +50,7 @@ export const toxic: IntermediateModerationPhase = async ({
 
   let endpoint = integration.endpoint;
   if (isNil(endpoint)) {
-    // TODO: (wyattjoh) replace hardcoded default with config.
-    endpoint = "https://commentanalyzer.googleapis.com/v1alpha1";
+    endpoint = TOXICITY_ENDPOINT_DEFAULT;
 
     log.trace(
       { endpoint },
@@ -79,7 +79,7 @@ export const toxic: IntermediateModerationPhase = async ({
   }
 
   // TODO: (wyattjoh) replace hardcoded default with config.
-  const timeout = ms("500ms");
+  const timeout = ms("800ms");
 
   try {
     logger.trace("checking comment toxicity");
@@ -125,6 +125,13 @@ export const toxic: IntermediateModerationPhase = async ({
     }
 
     log.trace({ score, isToxic, threshold }, "comment was not toxic");
+
+    return {
+      metadata: {
+        // Store the scores from perspective in the Comment metadata.
+        perspective: { model, score },
+      },
+    };
   } catch (err) {
     // Rethrow any ToxicCommentError.
     if (err instanceof ToxicCommentError) {