[CORL-810] Custom Moderation Phases (#2901)

* feat: initial implementation

* feat: renamed fields from mutations

* fix: more renaming to streamline {Key,Secret}->SigningSecret

* feat: introduced WrappedInternalError

* feat: enhanced extern payload, more fetch options

- Added tenant.{id,domain} to extern payload
- Added site.id to the extern payload
- Added response size limit to fetch
- Added new SCRAPE_MAX_RESPONSE_SIZE env var for managing the size of
  responses for scraping

* fix: fixed bug with scrape invocation

* feat: added more queries + mutations

- Added Query.externalModerationPhase
- Added Mutation.createExternalModerationPhase
- Added Mutation.updateExternalModerationPhase
- Added Mutation.enableExternalModerationPhase
- Added Mutation.disableExternalModerationPhase
- Added Mutation.deleteExternalModerationPhase
- Added Mutation.rotateExternalModerationPhaseSigningSecret

* feat: added secret management

* fix: linting

* fix: merge conflict fix

* feat: added UI

* fix: linting

* fix: linting

* fix: updated snapshots

* fix: improved docs

* fix: improved docs

* fix: added locales

* review: improve naming

* review: some review changes

- Switched /moderation/phase to /moderation/phases
- Fixed scrolling
- Fixed redirection

* fix: added scroll timeout for webhooks

src/core/server/queue/index.ts

@@ -5,7 +5,7 @@ import { Config } from "coral-server/config";
 import { I18n } from "coral-server/services/i18n";
 import { JWTSigningConfig } from "coral-server/services/jwt";
 import { AugmentedRedis, createRedisClient } from "coral-server/services/redis";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 
 import { createMailerTask, MailerQueue } from "./tasks/mailer";
 import { createNotifierTask, NotifierQueue } from "./tasks/notifier";

src/core/server/queue/tasks/mailer/content.ts

@@ -3,8 +3,10 @@ import nunjucks, { Environment, ILoader } from "nunjucks";
 import path from "path";
 
 import { Config } from "coral-server/config";
-import TenantCache from "coral-server/services/tenant/cache";
-import { TenantCacheAdapter } from "coral-server/services/tenant/cache/adapter";
+import {
+  TenantCache,
+  TenantCacheAdapter,
+} from "coral-server/services/tenant/cache";
 
 import { Tenant } from "coral-server/models/tenant";
 import { EmailTemplate } from "./templates";

src/core/server/queue/tasks/mailer/index.ts

@@ -4,7 +4,7 @@ import { createTimer } from "coral-server/helpers";
 import logger from "coral-server/logger";
 import Task from "coral-server/queue/Task";
 import MailerContent from "coral-server/queue/tasks/mailer/content";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 
 import {
   createJobProcessor,

src/core/server/queue/tasks/mailer/processor.ts

@@ -12,15 +12,17 @@ import { Db } from "mongodb";
 import { createTransport } from "nodemailer";
 import { Options } from "nodemailer/lib/smtp-connection";
 
-import { LanguageCode } from "coral-common/helpers/i18n/locales";
+import { LanguageCode } from "coral-common/helpers";
 import { Config } from "coral-server/config";
-import { InternalError } from "coral-server/errors";
+import { WrappedInternalError } from "coral-server/errors";
 import { createTimer } from "coral-server/helpers";
 import logger from "coral-server/logger";
 import { Tenant } from "coral-server/models/tenant";
 import { I18n, translate } from "coral-server/services/i18n";
-import TenantCache from "coral-server/services/tenant/cache";
-import { TenantCacheAdapter } from "coral-server/services/tenant/cache/adapter";
+import {
+  TenantCache,
+  TenantCacheAdapter,
+} from "coral-server/services/tenant/cache";
 
 export const JOB_NAME = "mailer";
 
@@ -254,7 +256,7 @@ export const createJobProcessor = (options: MailProcessorOptions) => {
         data
       );
     } catch (e) {
-      throw new InternalError(e, "could not translate the message");
+      throw new WrappedInternalError(e, "could not translate the message");
     }
 
     log.trace(
@@ -284,7 +286,7 @@ export const createJobProcessor = (options: MailProcessorOptions) => {
         // Create the transport based on the smtp uri.
         transport = createTransport(opts);
       } catch (e) {
-        throw new InternalError(e, "could not create email transport");
+        throw new WrappedInternalError(e, "could not create email transport");
       }
 
       // Set the transport back into the cache.
@@ -303,7 +305,7 @@ export const createJobProcessor = (options: MailProcessorOptions) => {
       // Send the mail message.
       await transport.sendMail(message);
     } catch (e) {
-      throw new InternalError(e, "could not send email");
+      throw new WrappedInternalError(e, "could not send email");
     }
 
     log.debug({ responseTime: messageSendTimer() }, "sent the email");

src/core/server/queue/tasks/notifier/index.ts

@@ -10,7 +10,7 @@ import {
   categories,
   NotificationCategory,
 } from "coral-server/services/notifications/categories";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 
 import { createJobProcessor, JOB_NAME, NotifierData } from "./processor";
 

src/core/server/queue/tasks/notifier/processor.ts

@@ -10,7 +10,7 @@ import { JWTSigningConfig } from "coral-server/services/jwt";
 import { NotificationCategory } from "coral-server/services/notifications/categories";
 import NotificationContext from "coral-server/services/notifications/context";
 import { Notification } from "coral-server/services/notifications/notification";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 
 import {
   filterSuperseded,

src/core/server/queue/tasks/rejector.ts

@@ -11,7 +11,7 @@ import {
 import { Connection } from "coral-server/models/helpers";
 import Task from "coral-server/queue/Task";
 import { AugmentedRedis } from "coral-server/services/redis";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 import { rejectComment } from "coral-server/stacks";
 
 import { GQLCOMMENT_SORT } from "coral-server/graph/schema/__generated__/types";

src/core/server/queue/tasks/webhook/processor.ts

@@ -1,4 +1,3 @@
-import crypto from "crypto";
 import { Redis } from "ioredis";
 import { Db } from "mongodb";
 
@@ -6,19 +5,15 @@ import { Config } from "coral-server/config";
 import { CoralEventPayload } from "coral-server/events/event";
 import { createTimer } from "coral-server/helpers";
 import logger from "coral-server/logger";
+import { filterExpiredSigningSecrets } from "coral-server/models/settings";
 import {
-  filterActiveSecrets,
-  filterExpiredSecrets,
-} from "coral-server/models/settings";
-import {
-  deleteEndpointSecrets,
-  Endpoint,
+  deleteTenantWebhookEndpointSigningSecrets,
   getWebhookEndpoint,
 } from "coral-server/models/tenant";
 import { JobProcessor } from "coral-server/queue/Task";
-import { createFetch, FetchOptions } from "coral-server/services/fetch";
+import { createFetch, generateFetchOptions } from "coral-server/services/fetch";
 import { disableWebhookEndpoint } from "coral-server/services/tenant";
-import TenantCache from "coral-server/services/tenant/cache";
+import { TenantCache } from "coral-server/services/tenant/cache";
 
 export const JOB_NAME = "webhook";
 
@@ -53,36 +48,7 @@ export interface WebhookDelivery {
   createdAt: Date;
 }
 
-/**
- * generateSignature will generate a signature used to assist clients to
- * validate that the request came from Coral.
- *
- * @param secret the secret used to sign the body with
- * @param body the body to use when signing
- */
-export function generateSignature(secret: string, body: string) {
-  return crypto
-    .createHmac("sha256", secret)
-    .update(body)
-    .digest()
-    .toString("hex");
-}
-
-export function generateSignatures(
-  endpoint: Pick<Endpoint, "signingSecrets">,
-  body: string,
-  now: Date
-) {
-  // For each of the signatures, we only want to sign the body with secrets that
-  // are still active.
-  return endpoint.signingSecrets
-    .filter(filterActiveSecrets(now))
-    .map(({ secret }) => generateSignature(secret, body))
-    .map((signature) => `sha256=${signature}`)
-    .join(",");
-}
-
-type CoralWebhookEventPayload = CoralEventPayload & {
+interface CoralWebhookEventPayload extends CoralEventPayload {
   /**
    * tenantID is the ID of the Tenant that this event originated at.
    */
@@ -92,28 +58,6 @@ type CoralWebhookEventPayload = CoralEventPayload & {
    * tenantDomain is the domain that is associated with this Tenant that this event originated at.
    */
   readonly tenantDomain: string;
-};
-
-export function generateFetchOptions(
-  endpoint: Pick<Endpoint, "signingSecrets">,
-  data: CoralWebhookEventPayload,
-  now: Date
-): FetchOptions {
-  // Serialize the body and signature to include in the request.
-  const body = JSON.stringify(data, null, 2);
-  const signature = generateSignatures(endpoint, body, now);
-
-  const headers: Record<string, any> = {
-    "Content-Type": "application/json",
-    "X-Coral-Event": data.type,
-    "X-Coral-Signature": signature,
-  };
-
-  return {
-    method: "POST",
-    headers,
-    body,
-  };
 }
 
 export function createJobProcessor({
@@ -162,12 +106,21 @@ export function createJobProcessor({
     // Get the current date.
     const now = new Date();
 
+    // Generate the payload.
+    const payload: CoralWebhookEventPayload = {
+      ...event,
+      tenantID,
+      tenantDomain: tenant.domain,
+    };
+
     // Get the fetch options.
-    const options = generateFetchOptions(
-      endpoint,
-      { ...event, tenantID, tenantDomain: tenant.domain },
-      now
-    );
+    const options = generateFetchOptions(endpoint.signingSecrets, payload, now);
+
+    // Add the X-Coral-Event header.
+    options.headers = {
+      ...options.headers,
+      "X-Coral-Event": event.type,
+    };
 
     // Send the request.
     const timer = createTimer();
@@ -240,35 +193,34 @@ export function createJobProcessor({
           tenant,
           endpointID
         );
-      } else {
-        // TODO: (wyattjoh) maybe schedule a retry?
       }
+      // TODO: (wyattjoh) maybe schedule a retry?
     }
 
     // Remove the expired secrets in the next tick so that it does not affect
     // the sending performance of this job, and errors do not impact the
     // sending.
-    const expiredSigningSecrets = endpoint.signingSecrets.filter(
-      filterExpiredSecrets(now)
-    );
-    if (expiredSigningSecrets.length > 0) {
+    const expiredSigningSecretKIDs = endpoint.signingSecrets
+      .filter(filterExpiredSigningSecrets(now))
+      .map((s) => s.kid);
+    if (expiredSigningSecretKIDs.length > 0) {
       process.nextTick(() => {
-        deleteEndpointSecrets(
+        deleteTenantWebhookEndpointSigningSecrets(
           mongo,
           tenantID,
           endpoint.id,
-          expiredSigningSecrets.map((s) => s.kid)
+          expiredSigningSecretKIDs
         )
           .then(() => {
             log.info(
-              { secrets: expiredSigningSecrets.length },
+              { endpointID: endpoint.id, kids: expiredSigningSecretKIDs },
               "removed expired secrets from endpoint"
             );
           })
           .catch((err) => {
             log.error(
               { err },
-              "an error occurred when trying to remove expired secrets"
+              "an error occurred when trying to remove expired endpoint secrets"
             );
           });
       });