[CORL-498, CORL-495, CORL-539, CORL-496, CORL-494] Email Notifications Support & Framework (#2498)

* chore: renamed old templates

* feat: initial notifications support

* feat: email enhancements

* fix: linting

* feat: initial digesting beheviour

* feat: added notification configuration

* feat: added unsubscribe routes

* fix: fixed failing snapshots/tests bc random ids

* feat: adjusted the save beheviour, added tests

* feat: added tests

* feat: added staff replies

* feat: renamed E-Mail to Email

* feat: enhanced cron processing

* fix: linting + updating tests

* feat: enhanced cron context

* fix: added staff replies back in

src/core/server/queue/index.ts

@@ -2,15 +2,15 @@ import Queue from "bull";
 import { Db } from "mongodb";
 
 import { Config } from "coral-server/config";
-import { createMailerTask, MailerQueue } from "coral-server/queue/tasks/mailer";
-import {
-  createScraperTask,
-  ScraperQueue,
-} from "coral-server/queue/tasks/scraper";
 import { I18n } from "coral-server/services/i18n";
+import { JWTSigningConfig } from "coral-server/services/jwt";
 import { createRedisClient } from "coral-server/services/redis";
 import TenantCache from "coral-server/services/tenant/cache";
 
+import { createMailerTask, MailerQueue } from "./tasks/mailer";
+import { createNotifierTask, NotifierQueue } from "./tasks/notifier";
+import { createScraperTask, ScraperQueue } from "./tasks/scraper";
+
 const createQueueOptions = async (
   config: Config
 ): Promise<Queue.QueueOptions> => {
@@ -46,11 +46,13 @@ export interface QueueOptions {
   config: Config;
   tenantCache: TenantCache;
   i18n: I18n;
+  signingConfig: JWTSigningConfig;
 }
 
 export interface TaskQueue {
   mailer: MailerQueue;
   scraper: ScraperQueue;
+  notifier: NotifierQueue;
 }
 
 export async function createQueue(options: QueueOptions): Promise<TaskQueue> {
@@ -61,10 +63,15 @@ export async function createQueue(options: QueueOptions): Promise<TaskQueue> {
   // Attach process functions to the various tasks in the queue.
   const mailer = createMailerTask(queueOptions, options);
   const scraper = createScraperTask(queueOptions, options);
+  const notifier = createNotifierTask(queueOptions, {
+    mailerQueue: mailer,
+    ...options,
+  });
 
   // Return the tasks + client.
   return {
     mailer,
     scraper,
+    notifier,
   };
 }

src/core/server/queue/tasks/Task.ts

@@ -46,13 +46,20 @@ export default class Task<T, U = any> {
         "processing job from queue"
       );
 
-      // Send the job off to the job processor to be handled.
-      const promise: U = await this.options.jobProcessor(job);
-      logger.trace(
-        { jobID: job.id, jobName: this.options.jobName },
-        "processing completed"
-      );
-      return promise;
+      try {
+        // Send the job off to the job processor to be handled.
+        const promise: U = await this.options.jobProcessor(job);
+        logger.trace(
+          { jobID: job.id, jobName: this.options.jobName },
+          "processing completed"
+        );
+
+        return promise;
+      } catch (err) {
+        logger.error({ err }, "failed to process job from queue");
+
+        throw err;
+      }
     });
 
     logger.trace(

src/core/server/queue/tasks/mailer/README.md

@@ -0,0 +1,37 @@
+# mailer
+
+The mailer is responsible for rendering and translating all email messages sent
+by Coral.
+
+## Adding a new email
+
+The first step to defining your new email is to add a new email template to
+`src/core/server/queue/tasks/mailer/templates/index.ts`. There you can see how
+other templates define their requirements, and how you ensure that you get the
+required context.
+
+### Templates
+
+Depending on the type of email you're adding, you're likely adding a:
+
+- _Account Notification_ - an email sent as a result of an account action by the
+  system or an administrator.
+- _Notification_ - an email sent to a user based on a notification preference
+  they've enabled.
+
+There are folders under `templates` for each of these. Use any of the templates
+in those folders as an example to craft your own template.
+
+### Translations
+
+Once you've added a template, you need to add a translation into the `src/core/server/locales/${locale}/email.ftl`
+file. There are two translation lines you need to add to support a new email:
+
+- _Subject_ - the subject of the email to be sent. Keys for these translations
+  follow the form `email-subject-${camelCase(templateName)}`.
+  - For example, for the template name `account-notification/confirm-email`
+    the subject translation key would become `email-subject-accountNotificationConfirmEmail`.
+- _Template_ - the translated body of the email to be sent. Keys for these
+  translations follow the form: `email-template-${camelCase(templateName)}`.
+  - For example, for the template name `account-notification/confirm-email`
+    the subject translation key would become `email-template-accountNotificationConfirmEmail`.

src/core/server/queue/tasks/mailer/assets/main.css

@@ -12,3 +12,7 @@ table {
   padding: 10px;
   background-color: whitesmoke;
 }
+
+.footer {
+  text-align: center;
+}

src/core/server/queue/tasks/mailer/content.ts

@@ -7,7 +7,7 @@ import TenantCache from "coral-server/services/tenant/cache";
 import { TenantCacheAdapter } from "coral-server/services/tenant/cache/adapter";
 
 import { Tenant } from "coral-server/models/tenant";
-import { Template } from "./templates";
+import { EmailTemplate } from "./templates";
 
 // templateDirectory is the directory containing the email templates.
 const templateDirectory = path.join(__dirname, "templates");
@@ -23,11 +23,11 @@ export interface MailerContentOptions {
  * @param env the nunjucks rendering environment
  * @param template the template to render
  */
-function render(env: Environment, { name, context }: Template) {
+function render(env: Environment, { name, context }: EmailTemplate) {
   return new Promise<string>((resolve, reject) =>
     env.render(
       name + ".html",
-      { context, name: camelCase(name) },
+      { context, name: camelCase(name), baseName: path.basename(name) },
       (err, html) => {
         if (err) {
           return reject(err);
@@ -90,7 +90,7 @@ export default class MailerContent {
    */
   public async generateHTML(
     tenant: Tenant,
-    template: Template
+    template: EmailTemplate
   ): Promise<string> {
     // Get the environment to render with.
     const env = this.getEnvironment(tenant);

src/core/server/queue/tasks/mailer/index.ts

@@ -12,13 +12,13 @@ import {
   MailerData,
   MailProcessorOptions,
 } from "./processor";
-import { Template } from "./templates";
+import { EmailTemplate } from "./templates";
 
 export interface MailerInput {
   message: {
     to: string;
   };
-  template: Template;
+  template: EmailTemplate;
   tenantID: string;
 }
 

src/core/server/queue/tasks/mailer/processor.ts

@@ -1,4 +1,5 @@
 import { Job } from "bull";
+import createDOMPurify from "dompurify";
 import { DOMLocalization } from "fluent-dom/compat";
 import { FluentBundle } from "fluent/compat";
 import { minify } from "html-minifier";
@@ -16,6 +17,7 @@ import { LanguageCode } from "coral-common/helpers/i18n/locales";
 import { Config } from "coral-server/config";
 import { InternalError } from "coral-server/errors";
 import logger from "coral-server/logger";
+import { Tenant } from "coral-server/models/tenant";
 import { I18n, translate } from "coral-server/services/i18n";
 import TenantCache from "coral-server/services/tenant/cache";
 import { TenantCacheAdapter } from "coral-server/services/tenant/cache/adapter";
@@ -98,6 +100,7 @@ function createMessageTranslator(i18n: I18n) {
    * @param data data used to send the message
    */
   return async (
+    tenant: Tenant,
     templateName: string,
     locale: LanguageCode,
     fromAddress: string,
@@ -121,22 +124,35 @@ function createMessageTranslator(i18n: I18n) {
     // Translate the bundle.
     await loc.translateFragment(dom.window.document);
 
-    // TODO: (wyattjoh) strip the i18n attributes from the source.
-
     // Grab the rendered HTML from the dom, and juice them.
     if (!dom.window.document.documentElement) {
       throw new Error("dom did not have a document element");
     }
-    const translatedHTML = dom.window.document.documentElement.outerHTML;
+
+    // Configure the purification.
+    const purify = createDOMPurify<false>(dom.window);
+
+    // Strip the l10n attributes from the email HTML.
+    purify.sanitize(dom.window.document.documentElement, {
+      ALLOW_DATA_ATTR: false,
+      WHOLE_DOCUMENT: true,
+      SANITIZE_DOM: false,
+      RETURN_DOM: false,
+      ADD_TAGS: ["link"],
+      FORBID_TAGS: [],
+      FORBID_ATTR: [],
+      IN_PLACE: true,
+    });
 
     // Juice the HTML to inline resources.
-    const html = await juiceHTML(translatedHTML);
+    const html = await juiceHTML(dom.serialize());
 
     // Get the translated subject.
     const subject = translate(
       bundle,
       templateName,
-      `email-subject-${camelCase(templateName)}`
+      `email-subject-${camelCase(templateName)}`,
+      { organizationName: tenant.organization.name }
     );
 
     // Generate the text content of the message from the HTML.
@@ -231,6 +247,7 @@ export const createJobProcessor = (options: MailProcessorOptions) => {
     let message: Message;
     try {
       message = await translateMessage(
+        tenant,
         data.templateName,
         tenant.locale,
         fromAddress,

src/core/server/queue/tasks/mailer/templates/account-notification/ban.html

@@ -0,0 +1,11 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationBan" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br/><br/>
+    Someone with access to your account has violated our community guidelines.
+    As a result, your account has been banned. You will no longer be able to
+    comment, react or report comments. If you think this has been done in error,
+    please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/confirm-email.html

@@ -0,0 +1,11 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationConfirmEmail" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br/><br/>
+    To confirm your email address for use with your commenting account at {{ context.organizationName }},
+    please follow this link: <a data-l10n-name="confirmYourEmail" href="{{ context.confirmURL }}">Click here to confirm your email</a><br/><br/>
+    If you did not recently create a commenting account with
+    {{ context.organizationName }}, you can safely ignore this email.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/delete-request-cancel.html

@@ -1,4 +1,4 @@
-{% extends "layouts/user-notification.html" %}
+{% extends "layouts/account-notification.html" %}
 
 {% block content %}
   You have cancelled your account deletion request for {{ context.organizationName }}. Your account is now reactivated.

src/core/server/queue/tasks/mailer/templates/account-notification/delete-request-completed.html

@@ -1,4 +1,4 @@
-{% extends "layouts/user-notification.html" %}
+{% extends "layouts/account-notification.html" %}
 
 {% block content %}
   Your commenter account for {{ context.organizationName }} is now deleted. We're sorry to see you go!

src/core/server/queue/tasks/mailer/templates/account-notification/delete-request-confirmation.html

@@ -1,4 +1,4 @@
-{% extends "layouts/user-notification.html" %}
+{% extends "layouts/account-notification.html" %}
 
 {% block content %}
   A request to delete your commenter account was received. Your account is scheduled for deletion on {{ context.requestDate }}.

src/core/server/queue/tasks/mailer/templates/account-notification/download-comments.html

@@ -0,0 +1,9 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationDownloadComments" data-l10n-args="{{ context | dump }}">
+    Your comments from {{ context.organizationName }} as of {{ context.date }} are
+    now available for download.<br /><br />
+    <a data-l10n-name="downloadUrl" href="{{ context.downloadUrl }}">Download my comment archive</a>
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/forgot-password.html

@@ -0,0 +1,10 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationForgotPassword" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br/><br/>
+    We received a request to reset your password on <a data-l10n-name="organizationName" href="{{ context.organizationURL }}"></a>.<br/><br/>
+    Please follow this link reset your password: <a data-l10n-name="resetYourPassword" href="{{ context.resetURL }}">Click here to reset your password</a><br/><br/>
+    <i>If you did not request this, you can ignore this email.</i><br/>
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/invite.html

@@ -0,0 +1,8 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationInvite" data-l10n-args="{{ context | dump }}">
+    You have been invited to join the {{ context.organizationName }} team on Coral. Finish
+    setting up your account <a data-l10n-name="invite" href="{{ context.inviteURL }}">here</a>.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/password-change.html

@@ -0,0 +1,10 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationPasswordChange" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br/><br/>
+    The password on your account has been changed.<br/><br/>
+    If you did not request this change,
+    please contact please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/suspend.html

@@ -0,0 +1,15 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationSuspend" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br/><br/>
+
+    In accordance with {{ context.organizationName }}'s community guidelines, your
+    account has been temporarily suspended. During the suspension, you will be
+    unable to comment, flag or engage with fellow commenters. Please rejoin the
+    conversation {{ context.until }}.<br/><br/>
+
+    If you think this has been done in error, please contact our community team at
+    <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/account-notification/update-username.html

@@ -0,0 +1,11 @@
+{% extends "layouts/account-notification.html" %}
+
+{% block content %}
+  <div data-l10n-id="email-template-accountNotificationUpdateUsername" data-l10n-args="{{ context | dump }}">
+    Hello {{ context.username }},<br /><br />
+
+    Thank you for updating your {{ context.organizationName }} commenter account
+    information. The changes you made are effective immediately. If you did not make
+    this change please reach out to <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/ban.html

@@ -1,9 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Hello {{ context.username }},<br/><br/>
-  Someone with access to your account has violated our community guidelines.
-  As a result, your account has been banned. You will no longer be able to
-  comment, react or report comments. If you think this has been done in error,
-  please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/confirm-email.html

@@ -1,9 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Hello {{ context.username }},<br/><br/>
-  To confirm your email address for use with your commenting account at {{ context.organizationName }},
-  please follow this link: <a data-l10n-name="confirmYourEmail" href="{{ context.confirmURL }}">Click here to confirm your email</a><br/><br/>
-  If you did not recently create a commenting account with
-  {{ context.organizationName }}, you can safely ignore this email.
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/download-comments.html

@@ -1,7 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Your comments from {{ context.organizationName }} as of {{ context.date }} are
-  now available for download.<br /><br />
-  <a data-l10n-name="downloadUrl" href="{{ context.downloadUrl }}">Download my comment archive</a>
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/forgot-password.html

@@ -1,8 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Hello {{ context.username }},<br/><br/>
-  We received a request to reset your password on <a data-l10n-name="organizationName" href="{{ context.organizationURL }}"></a>.<br/><br/>
-  Please follow this link reset your password: <a data-l10n-name="resetYourPassword" href="{{ context.resetURL }}">Click here to reset your password</a><br/><br/>
-  <i>If you did not request this, you can ignore this email.</i><br/>
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/index.ts

@@ -1,9 +1,67 @@
-interface Template<T extends string, U extends {}> {
+interface EmailTemplate<T extends string, U extends {}> {
   name: T;
   context: U;
 }
 
-type UserNotificationContext<T extends string, U extends {}> = Template<
+/**
+ * NotificationContext
+ */
+
+type NotificationContext<T extends string, U extends {}> = EmailTemplate<
+  T,
+  U & {
+    organizationURL: string;
+    organizationName: string;
+    unsubscribeURL: string;
+  }
+>;
+
+export type OnReplyTemplate = NotificationContext<
+  "notification/on-reply",
+  {
+    storyTitle: string;
+    storyURL: string;
+    authorUsername: string;
+    commentPermalink: string;
+  }
+>;
+
+export type OnStaffReplyTemplate = NotificationContext<
+  "notification/on-staff-reply",
+  {
+    storyTitle: string;
+    storyURL: string;
+    authorUsername: string;
+    commentPermalink: string;
+  }
+>;
+
+export type DigestibleTemplate = OnReplyTemplate | OnStaffReplyTemplate;
+
+type DigestTemplate = NotificationContext<
+  "notification/digest",
+  {
+    digests: Array<{
+      /**
+       * partial stores the part of the filename that can be used to tie the
+       * given notification into a specific template.
+       */
+      partial: string;
+
+      /**
+       * contexts is the array of all the contexts under this partial that
+       * should be used to create the digest context.
+       */
+      contexts: Array<DigestibleTemplate["context"]>;
+    }>;
+  }
+>;
+
+/**
+ * AccountNotificationContext
+ */
+
+type AccountNotificationContext<T extends string, U extends {}> = EmailTemplate<
   T,
   U & {
     organizationURL: string;
@@ -11,16 +69,16 @@ type UserNotificationContext<T extends string, U extends {}> = Template<
   }
 >;
 
-export type ForgotPasswordTemplate = UserNotificationContext<
-  "forgot-password",
+export type ForgotPasswordTemplate = AccountNotificationContext<
+  "account-notification/forgot-password",
   {
     username: string;
     resetURL: string;
   }
 >;
 
-export type BanTemplate = UserNotificationContext<
-  "ban",
+export type BanTemplate = AccountNotificationContext<
+  "account-notification/ban",
   {
     username: string;
     organizationContactEmail: string;
@@ -28,8 +86,8 @@ export type BanTemplate = UserNotificationContext<
   }
 >;
 
-export type SuspendTemplate = UserNotificationContext<
-  "suspend",
+export type SuspendTemplate = AccountNotificationContext<
+  "account-notification/suspend",
   {
     username: string;
     until: string;
@@ -38,16 +96,16 @@ export type SuspendTemplate = UserNotificationContext<
   }
 >;
 
-export type PasswordChangeTemplate = UserNotificationContext<
-  "password-change",
+export type PasswordChangeTemplate = AccountNotificationContext<
+  "account-notification/password-change",
   {
     username: string;
     organizationContactEmail: string;
   }
 >;
 
-export type ConfirmEmailTemplate = UserNotificationContext<
-  "confirm-email",
+export type ConfirmEmailTemplate = AccountNotificationContext<
+  "account-notification/confirm-email",
   {
     username: string;
     confirmURL: string;
@@ -55,15 +113,15 @@ export type ConfirmEmailTemplate = UserNotificationContext<
   }
 >;
 
-export type InviteEmailTemplate = UserNotificationContext<
-  "invite",
+export type InviteEmailTemplate = AccountNotificationContext<
+  "account-notification/invite",
   {
     inviteURL: string;
   }
 >;
 
-export type DownloadCommentsTemplate = UserNotificationContext<
-  "download-comments",
+export type DownloadCommentsTemplate = AccountNotificationContext<
+  "account-notification/download-comments",
   {
     username: string;
     date: string;
@@ -71,33 +129,37 @@ export type DownloadCommentsTemplate = UserNotificationContext<
   }
 >;
 
-export type UpdateUsernameTemplate = UserNotificationContext<
-  "update-username",
+export type UpdateUsernameTemplate = AccountNotificationContext<
+  "account-notification/update-username",
   {
     username: string;
     organizationContactEmail: string;
   }
 >;
 
-export type AccountDeletionConfirmation = UserNotificationContext<
-  "delete-request-confirmation",
+export type AccountDeletionConfirmation = AccountNotificationContext<
+  "account-notification/delete-request-confirmation",
   {
     requestDate: string;
   }
 >;
 
-export type AccountDeletionCancellation = UserNotificationContext<
-  "delete-request-cancel",
+export type AccountDeletionCancellation = AccountNotificationContext<
+  "account-notification/delete-request-cancel",
   {}
 >;
 
-export type AccountDeletionCompleted = UserNotificationContext<
-  "delete-request-completed",
+export type AccountDeletionCompleted = AccountNotificationContext<
+  "account-notification/delete-request-completed",
   {
     organizationContactEmail: string;
   }
 >;
 
+/**
+ * Templates
+ */
+
 type Templates =
   | BanTemplate
   | ConfirmEmailTemplate
@@ -109,6 +171,9 @@ type Templates =
   | UpdateUsernameTemplate
   | AccountDeletionConfirmation
   | AccountDeletionCancellation
-  | AccountDeletionCompleted;
+  | AccountDeletionCompleted
+  | OnReplyTemplate
+  | OnStaffReplyTemplate
+  | DigestTemplate;
 
-export { Templates as Template };
+export { Templates as EmailTemplate };

src/core/server/queue/tasks/mailer/templates/invite.html

@@ -1,6 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  You have been invited to join the {{ context.organizationName }} team on Coral. Finish
-  setting up your account <a data-l10n-name="invite" href="{{ context.inviteURL }}">here</a>.
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/layouts/account-notification.html

@@ -0,0 +1,7 @@
+{% extends "layouts/base.html" %}
+
+{% block footer %}
+  <div data-l10n-id="email-footer-accountNotification" data-l10n-args="{{ context | dump }}">
+    Sent by <a data-l10n-name="organizationLink" href="{{ context.organizationURL }}">{{ context.organizationName }}</a>
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/layouts/base.html

@@ -9,6 +9,17 @@
     <link href="assets/main.css" rel="stylesheet">
   </head>
   <body>
-    {% block body %}{% endblock %}
+    <table>
+      <tr>
+        <td class="content">
+          {% block content %}{% endblock %}
+        </td>
+      </tr>
+      <tr>
+        <td class="footer">
+          {% block footer %}{% endblock %}
+        </td>
+      </tr>
+    </table>
   </body>
 </html>

src/core/server/queue/tasks/mailer/templates/layouts/notification.html

@@ -0,0 +1,11 @@
+{% extends "layouts/base.html" %}
+
+{% block content %}
+  {% include "notification/partials/" + baseName + ".html" %}
+{% endblock %}
+
+{% block footer %}
+  <div data-l10n-id="email-footer-notification" data-l10n-args="{{ context | dump }}">
+    Sent by <a data-l10n-name="organizationLink" href="{{ context.organizationURL }}">{{ context.organizationName }}</a> - <a data-l10n-name="unsubscribeLink" href="{{ context.unsubscribeURL }}">Unsubscribe from these notifications</a>
+  </div>
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/layouts/user-notification.html

@@ -1,16 +0,0 @@
-{% extends "layouts/base.html" %}
-
-{% block body %}
-  <table>
-    <tr>
-      <td class="content" data-l10n-id="email-notification-template-{{ name }}" data-l10n-args="{{ context | dump }}">
-        {% block content %}{% endblock %}
-      </td>
-    </tr>
-    <tr>
-      <td align="center" data-l10n-id="email-notification-footer" data-l10n-args="{{ context | dump }}">
-        Sent by <a data-l10n-name="organizationLink" href="{{ context.organizationURL }}">{{ context.organizationName }}</a>
-      </td>
-    </tr>
-  </table>
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/notification/digest.html

@@ -0,0 +1,9 @@
+{% extends "layouts/notification.html" %}
+
+{% block content %}
+  {% for digest in context.digests %}
+    {% for context in digest.contexts %}
+      {% include "notification/partials/" + digest.partial + ".html" %}
+    {% endfor %}
+  {% endfor %}
+{% endblock %}

src/core/server/queue/tasks/mailer/templates/notification/on-reply.html

@@ -0,0 +1 @@
+{% extends "layouts/notification.html" %}

src/core/server/queue/tasks/mailer/templates/notification/on-staff-reply.html

@@ -0,0 +1 @@
+{% extends "layouts/notification.html" %}

src/core/server/queue/tasks/mailer/templates/notification/partials/on-reply.html

@@ -0,0 +1,4 @@
+<div data-l10n-id="email-template-notificationOnReply" data-l10n-args="{{ context | dump }}">
+  {{ context.organizationName }} - <a data-l10n-name="storyLink" href="{{ context.storyURL }}">{{ context.storyTitle }}</a><br /><br/>
+  {{ context.authorUsername }} has replied to your comment: <a data-l10n-name="commentPermalink" href="{{ context.commentPermalink }}">View comment</a>
+</div>

src/core/server/queue/tasks/mailer/templates/notification/partials/on-staff-reply.html

@@ -0,0 +1,4 @@
+<div data-l10n-id="email-template-notificationOnStaffReply" data-l10n-args="{{ context | dump }}">
+  {{ context.organizationName }} - <a data-l10n-name="storyLink" href="{{ context.storyURL }}">{{ context.storyTitle }}</a><br /><br/>
+  {{ context.authorUsername }} works for {{ context.organizationName }} and has replied to your comment: <a data-l10n-name="commentPermalink" href="{{ context.commentPermalink }}">View comment</a>
+</div>

src/core/server/queue/tasks/mailer/templates/password-change.html

@@ -1,8 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Hello {{ context.username }},<br/><br/>
-  The password on your account has been changed.<br/><br/>
-  If you did not request this change,
-  please contact please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/suspend.html

@@ -1,13 +0,0 @@
-{% extends "layouts/user-notification.html" %}
-
-{% block content %}
-  Hello {{ context.username }},<br/><br/>
-
-  In accordance with {{ context.organizationName }}'s community guidelines, your
-  account has been temporarily suspended. During the suspension, you will be
-  unable to comment, flag or engage with fellow commenters. Please rejoin the
-  conversation {{ context.until }}.<br/><br/>
-
-  If you think this has been done in error, please contact our community team at
-  <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
-{% endblock %}

src/core/server/queue/tasks/mailer/templates/update-username.html

@@ -1,7 +0,0 @@
-{% extends "layouts/user-notification.html" %} 
-{% block content %} Hello {{ context.username }},<br /><br />
-
-Thank you for updating your {{ context.organizationName }} commenter account
-information. The changes you made are effective immediately. If you did not make
-this change please reach out to <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>. 
-{% endblock %}

src/core/server/queue/tasks/notifier/index.ts

@@ -0,0 +1,71 @@
+import Queue from "bull";
+import { groupBy } from "lodash";
+import { Db } from "mongodb";
+
+import { Config } from "coral-server/config";
+import { SUBSCRIPTION_CHANNELS } from "coral-server/graph/tenant/resolvers/Subscription/types";
+import logger from "coral-server/logger";
+import Task from "coral-server/queue/Task";
+import { MailerQueue } from "coral-server/queue/tasks/mailer";
+import { JWTSigningConfig } from "coral-server/services/jwt";
+import {
+  categories,
+  NotificationCategory,
+} from "coral-server/services/notifications/categories";
+import TenantCache from "coral-server/services/tenant/cache";
+
+import { createJobProcessor, JOB_NAME, NotifierData } from "./processor";
+
+export const createNotifierTask = (
+  queue: Queue.QueueOptions,
+  options: Options
+) => new NotifierQueue(queue, options);
+
+interface Options {
+  mongo: Db;
+  mailerQueue: MailerQueue;
+  config: Config;
+  tenantCache: TenantCache;
+  signingConfig: JWTSigningConfig;
+}
+
+/**
+ * NotifierQueue is designed to handle creating and queuing notifications
+ * that could be sent to users.
+ */
+export class NotifierQueue {
+  private registry: Record<SUBSCRIPTION_CHANNELS, NotificationCategory[]>;
+  private task: Task<NotifierData>;
+
+  constructor(queue: Queue.QueueOptions, options: Options) {
+    // Notification categories have been grouped by their event name so that
+    // each event emitted need only access the associated notification once.
+    this.registry = groupBy(categories, "event") as Record<
+      SUBSCRIPTION_CHANNELS,
+      NotificationCategory[]
+    >;
+    this.task = new Task({
+      jobName: JOB_NAME,
+      jobProcessor: createJobProcessor({ registry: this.registry, ...options }),
+      queue,
+    });
+  }
+
+  public async add(data: NotifierData) {
+    // Get all the handlers that are active for this channel.
+    const c = this.registry[data.input.channel];
+    if (!c || c.length === 0) {
+      logger.debug(
+        { channel: data.input.channel },
+        "no notifications registered on this channel"
+      );
+      return;
+    }
+
+    return this.task.add(data);
+  }
+
+  public process() {
+    return this.task.process();
+  }
+}

src/core/server/queue/tasks/notifier/messages.spec.ts

@@ -0,0 +1,43 @@
+import { filterSuperseded, SupersededNotification } from "./messages";
+
+describe("filterSuperseded", () => {
+  it("handles when there is no superseded notifications", () => {
+    const notifications: SupersededNotification[] = [
+      {
+        category: { name: "staffReply", supersedesCategories: ["reply"] },
+        notification: { userID: "1" },
+      },
+      {
+        category: { name: "staffReply", supersedesCategories: ["reply"] },
+        notification: { userID: "1" },
+      },
+      {
+        category: { name: "staffReply", supersedesCategories: ["reply"] },
+        notification: { userID: "2" },
+      },
+    ];
+    expect(notifications.filter(filterSuperseded)).toHaveLength(3);
+  });
+
+  it("handles when there is superseded notifications", () => {
+    const notifications: SupersededNotification[] = [
+      {
+        category: { name: "staffReply", supersedesCategories: ["reply"] },
+        notification: { userID: "1" },
+      },
+      {
+        category: { name: "staffReply", supersedesCategories: ["reply"] },
+        notification: { userID: "1" },
+      },
+      {
+        category: { name: "reply", supersedesCategories: [] },
+        notification: { userID: "1" },
+      },
+      {
+        category: { name: "reply", supersedesCategories: [] },
+        notification: { userID: "2" },
+      },
+    ];
+    expect(notifications.filter(filterSuperseded)).toHaveLength(3);
+  });
+});

src/core/server/queue/tasks/notifier/messages.ts

@@ -0,0 +1,127 @@
+import { SUBSCRIPTION_INPUT } from "coral-server/graph/tenant/resolvers/Subscription/types";
+import { GQLDIGEST_FREQUENCY } from "coral-server/graph/tenant/schema/__generated__/types";
+import logger from "coral-server/logger";
+import { NotificationCategory } from "coral-server/services/notifications/categories";
+import NotificationContext from "coral-server/services/notifications/context";
+import { Notification } from "coral-server/services/notifications/notification";
+
+import { MailerQueue } from "../mailer";
+import { DigestibleTemplate } from "../mailer/templates";
+import { CategoryNotification } from "./processor";
+
+/**
+ * SupersededNotification is a subset of `CategoryNotification` to provide a
+ * minimal implementation.
+ */
+export interface SupersededNotification {
+  category: Pick<
+    CategoryNotification["category"],
+    "name" | "supersedesCategories"
+  >;
+  notification: Pick<CategoryNotification["notification"], "userID">;
+}
+
+/**
+ * filterSuperseded will filter all the possible notifications and only send
+ * those notifications that are not superseded by another type of notification.
+ */
+export const filterSuperseded = (
+  {
+    category: { name },
+    notification: { userID: destinationUserID },
+  }: SupersededNotification,
+  index: number,
+  notifications: SupersededNotification[]
+) =>
+  !notifications.some(
+    ({
+      category: { supersedesCategories = [] },
+      notification: { userID: notificationUserID },
+    }) =>
+      // Only allow notifications to supersede another notification if that
+      // notification is also destined for the same user.
+      notificationUserID === destinationUserID &&
+      // If another notification that is destined for the same user also exists
+      // and declares that it supersedes this one, return true so we can filter
+      // this one from the list.
+      supersedesCategories.some(
+        supersededCategory => supersededCategory === name
+      )
+  );
+
+export const handleHandlers = async (
+  ctx: NotificationContext,
+  categories: NotificationCategory[],
+  input: SUBSCRIPTION_INPUT
+): Promise<CategoryNotification[]> => {
+  const notifications: Array<CategoryNotification | null> = await Promise.all(
+    categories.map(async category => {
+      const notification = await category.process(ctx, input.payload);
+      if (!notification) {
+        return null;
+      }
+
+      return { category, notification };
+    })
+  );
+
+  // Filter out the categories that don't have notifications.
+  return notifications.filter(
+    notification => notification !== null
+  ) as CategoryNotification[];
+};
+
+/**
+ * processNewNotifications will handle notifications that are collected after
+ * an event hook. These notifications will be batched by user and optionally
+ * queued for digesting or sent immediately depending on the user's settings.
+ */
+export const processNewNotifications = async (
+  ctx: NotificationContext,
+  notifications: Notification[],
+  mailer: MailerQueue
+) => {
+  // Group all the notifications by user.
+  const userNotifications: Record<string, DigestibleTemplate[]> = {};
+  for (const { userID, template } of notifications) {
+    if (userID in userNotifications) {
+      userNotifications[userID].push(template);
+    } else {
+      userNotifications[userID] = [template];
+    }
+  }
+
+  // Load all the user's profiles into the context.
+  await ctx.users.loadMany(Object.keys(userNotifications));
+
+  // Send all the notifications for each user.
+  for (const [userID, templates] of Object.entries(userNotifications)) {
+    // Get the user from the context (which should have been already loaded from
+    // before).
+    const user = await ctx.users.load(userID);
+    if (!user) {
+      logger.warn(
+        { userID },
+        "attempted notification for user that wasn't found"
+      );
+      continue;
+    }
+
+    if (user.notifications.digestFrequency === GQLDIGEST_FREQUENCY.NONE) {
+      // Send the notifications for the user now, they don't have digesting
+      // enabled.
+      for (const template of templates) {
+        await mailer.add({
+          tenantID: ctx.tenant.id,
+          message: {
+            to: user.email!,
+          },
+          template,
+        });
+      }
+    } else {
+      // Queue up the notifications to be sent in the next user's digest.
+      await ctx.addDigests(user.id, templates);
+    }
+  }
+};

src/core/server/queue/tasks/notifier/processor.ts

@@ -0,0 +1,128 @@
+import { Job } from "bull";
+import { Db } from "mongodb";
+
+import { Config } from "coral-server/config";
+import {
+  SUBSCRIPTION_CHANNELS,
+  SUBSCRIPTION_INPUT,
+} from "coral-server/graph/tenant/resolvers/Subscription/types";
+import logger from "coral-server/logger";
+import { MailerQueue } from "coral-server/queue/tasks/mailer";
+import { JWTSigningConfig } from "coral-server/services/jwt";
+import { NotificationCategory } from "coral-server/services/notifications/categories";
+import NotificationContext from "coral-server/services/notifications/context";
+import { Notification } from "coral-server/services/notifications/notification";
+import TenantCache from "coral-server/services/tenant/cache";
+
+import {
+  filterSuperseded,
+  handleHandlers,
+  processNewNotifications,
+} from "./messages";
+
+export const JOB_NAME = "notifications";
+
+/**
+ * NotifierData stores the data used by the notification system.
+ */
+export interface NotifierData {
+  tenantID: string;
+  input: SUBSCRIPTION_INPUT;
+}
+
+interface Options {
+  mailerQueue: MailerQueue;
+  mongo: Db;
+  config: Config;
+  registry: Record<SUBSCRIPTION_CHANNELS, NotificationCategory[]>;
+  tenantCache: TenantCache;
+  signingConfig: JWTSigningConfig;
+}
+
+/**
+ * CategoryNotification combines the category and notification's to collect the
+ * appropriate elements together that can be used for digesting purposes.
+ */
+export interface CategoryNotification {
+  category: NotificationCategory;
+  notification: Notification;
+}
+
+/**
+ * createJobProcessor creates the processor that is used to process the
+ * possible notifications and queueing them up in the mailer if they need to be
+ * sent.
+ *
+ * @param options options for the processor
+ */
+export const createJobProcessor = ({
+  mailerQueue,
+  mongo,
+  config,
+  registry,
+  tenantCache,
+  signingConfig,
+}: Options) => {
+  return async (job: Job<NotifierData>) => {
+    const now = new Date();
+
+    // Pull the data out of the model.
+    const { tenantID, input } = job.data;
+
+    // Create a new logger to handle logging for this job.
+    const log = logger.child({
+      jobID: job.id,
+      jobName: JOB_NAME,
+      tenantID,
+    });
+
+    log.debug("starting to handle a notify operation");
+
+    try {
+      // Get all the handlers that are active for this channel.
+      const categories = registry[input.channel];
+      if (!categories || categories.length === 0) {
+        return;
+      }
+
+      // Grab the tenant from the cache.
+      const tenant = await tenantCache.retrieveByID(tenantID);
+      if (!tenant) {
+        throw new Error("tenant not found with ID");
+      }
+
+      // Create a notification context to handle processing notifications.
+      const ctx = new NotificationContext({
+        mongo,
+        config,
+        signingConfig,
+        tenant,
+        now,
+      });
+
+      // For each of the handler's we need to process, we should iterate to
+      // generate their notifications.
+      let notifications = await handleHandlers(ctx, categories, input);
+
+      // Check to see if some of the other notifications that are queued
+      // had this notification superseded.
+      notifications = notifications.filter(filterSuperseded);
+
+      // Send all the notifications now.
+      await processNewNotifications(
+        ctx,
+        notifications.map(({ notification }) => notification),
+        mailerQueue
+      );
+
+      log.debug(
+        { notifications: notifications.length },
+        "notifications handled"
+      );
+    } catch (err) {
+      log.error({ err }, "could not handle the notifications");
+
+      throw err;
+    }
+  };
+};

src/core/server/queue/tasks/scraper.ts

@@ -39,7 +39,6 @@ const createJobProcessor = ({ mongo }: ScrapeProcessorOptions) => async (
 
   try {
     await scrape(mongo, tenantID, storyID, storyURL);
-    log.debug("scraped the story");
   } catch (err) {
     log.error({ err }, "could not scrape the story");