[next] Email (#2261)

* feat: suspending, banning, now propogation

* feat: added email rendering + localization support

* fix: fix related to lib

* refactor: moved juicer to queue task

* refactor: cleanup of job processor

* refactor: improved error messaging around failed email

* feat: initial forgot passwor impl

* fix: fixed rebase errors

* feat: send back Content-Language header with requests

* feat: added ban email

* feat: implemented forgotten password API

* fix: linting

* feat: support more emails

* fix: promise patches

* feat: initial confirm email API

* feat: added rate limiting

* feat: added URL support

* feat: added email docs

* fix: updated docs

* chore: documentation review

* fix: fixed build bug

* feat: implement forgot password in auth popup

* test: add tests + fixes

* chore: rename StatelessComponent to FunctionComponent

* fix: types and test fixes

* chore: upgrade deps

* fix: THANK YOU TESTS FOR SAVING MY A**

* chore: reorder imports

* chore: remove obsolete !

* feat: implement accounts bundle

* refactor: review suggestion

* fix: rebase upgrade error

* fix: rebase bug

* feat: reset password link support

* test: add tests for account password reset page

* fix: remove redirect uri

* fix: revert local state changes
This commit is contained in:
Wyatt Johnson
2019-05-09 20:54:56 +00:00
committed by Kiwi
parent 945bd7f2b0
commit df57b4eb17
487 changed files with 6797 additions and 2434 deletions
+45 -13
View File
@@ -6,6 +6,7 @@ import logger from "talk-server/logger";
export interface TaskOptions<T, U = any> {
jobName: string;
jobProcessor: (job: Job<T>) => Promise<U>;
jobOptions?: Queue.JobOptions;
queue: Queue.QueueOptions;
}
@@ -14,10 +15,39 @@ export default class Task<T, U = any> {
private queue: QueueType<T>;
private log: Logger;
constructor(options: TaskOptions<T, U>) {
this.queue = new Queue(options.jobName, options.queue);
this.options = options;
this.log = logger.child({ jobName: options.jobName });
constructor({
jobName,
jobProcessor,
jobOptions = {},
queue,
}: TaskOptions<T, U>) {
this.log = logger.child({ jobName });
this.queue = new Queue(jobName, queue);
this.options = {
jobName,
jobProcessor,
jobOptions: {
// We always remove the job when it's complete, no need to fill up Redis
// with completed entries if we don't need to.
removeOnComplete: true,
// By default, configure jobs to use an exponential backoff
// strategy starting at a 10 second delay.
backoff: {
type: "exponential",
delay: 10000,
},
// Be default, try all jobs at least 5 times.
attempts: 5,
// Add the custom job options if they exist.
...jobOptions,
},
queue,
};
// TODO: (wyattjoh) attach event handlers to the queue for metrics via: https://github.com/OptimalBits/bull/blob/develop/REFERENCE.md#events
}
/**
@@ -27,11 +57,8 @@ export default class Task<T, U = any> {
* @param data the data for the job to add.
*/
public async add(data: T): Promise<Queue.Job<T> | undefined> {
const job = await this.queue.add(data, {
// We always remove the job when it's complete, no need to fill up Redis
// with completed entries if we don't need to.
removeOnComplete: true,
});
// Create the job.
const job = await this.queue.add(data, this.options.jobOptions);
this.log.trace({ jobID: job.id }, "added job to queue");
return job;
@@ -47,10 +74,15 @@ export default class Task<T, U = any> {
log.trace("processing job from queue");
// Send the job off to the job processor to be handled.
const promise: U = await this.options.jobProcessor(job);
log.trace("processing completed");
return promise;
try {
// Send the job off to the job processor to be handled.
const promise: U = await this.options.jobProcessor(job);
log.trace("processing completed");
return promise;
} catch (err) {
log.error({ err }, "job failed to process");
throw err;
}
});
this.log.trace("registered processor for job type");
+2
View File
@@ -7,6 +7,7 @@ import {
createScraperTask,
ScraperQueue,
} from "talk-server/queue/tasks/scraper";
import { I18n } from "talk-server/services/i18n";
import { createRedisClient } from "talk-server/services/redis";
import TenantCache from "talk-server/services/tenant/cache";
@@ -44,6 +45,7 @@ export interface QueueOptions {
mongo: Db;
config: Config;
tenantCache: TenantCache;
i18n: I18n;
}
export interface TaskQueue {
@@ -0,0 +1,14 @@
/*
* Refer to https://www.campaignmonitor.com/css/lists/list-style/ for supported
* features.
*/
.content {
padding: 10px;
background-color: white;
}
table {
padding: 10px;
background-color: whitesmoke;
}
+79 -21
View File
@@ -1,33 +1,84 @@
import nunjucks from "nunjucks";
import { camelCase } from "lodash";
import nunjucks, { Environment, ILoader } from "nunjucks";
import path from "path";
import { Config } from "talk-server/config";
import TenantCache from "talk-server/services/tenant/cache";
import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter";
/**
* templateDirectory is the directory containing the email templates.
*/
import { Tenant } from "talk-server/models/tenant";
import { Template } from "./templates";
// templateDirectory is the directory containing the email templates.
const templateDirectory = path.join(__dirname, "templates");
export interface GenerateHTMLOptions {
/**
* name is the name of the template to render.
*/
name: string;
context: any;
export interface MailerContentOptions {
config: Config;
tenantCache: TenantCache;
}
/**
* render will render the nunjucks template using the provided environment.
*
* @param env the nunjucks rendering environment
* @param template the template to render
*/
function render(env: Environment, { name, context }: Template) {
return new Promise<string>((resolve, reject) =>
env.render(
name + ".html",
{ context, name: camelCase(name) },
(err, html) => {
if (err) {
return reject(err);
}
return resolve(html);
}
)
);
}
export default class MailerContent {
private env: nunjucks.Environment;
private cache: TenantCacheAdapter<nunjucks.Environment>;
private config: Config;
private loaders: ILoader[];
constructor(config: Config) {
// Configure the nunjucks environment.
this.env = new nunjucks.Environment(
new nunjucks.FileSystemLoader(templateDirectory),
{
constructor({ config, tenantCache }: MailerContentOptions) {
this.config = config;
// Configure the environment cache.
this.cache = new TenantCacheAdapter(tenantCache);
// Configure the loaders for the templates that apply to all clients.
this.loaders = [
// Load the templates from the filesystem.
new nunjucks.FileSystemLoader(templateDirectory, {
// When we aren't in production mode, reload the templates.
watch: config.get("env") !== "production",
}
);
watch: this.config.get("env") !== "production",
}),
];
}
/**
* getEnvironment will get the environment from the cache if it exists, or
* create it, and add it to the cache otherwise.
*
* @param tenant the Tenant to generate the environment for
*/
private getEnvironment(tenant: Pick<Tenant, "id">): nunjucks.Environment {
// Get the nunjucks environment to use for generating the email HTML.
let env = this.cache.get(tenant.id);
if (!env) {
// TODO: (wyattjoh) add the custom loader per tenant here to support customizing templates.
// Configure the nunjucks environment.
env = new nunjucks.Environment(this.loaders);
// Set the environment in the cache.
this.cache.set(tenant.id, env);
}
return env;
}
/**
@@ -37,7 +88,14 @@ export default class MailerContent {
* @param options configuration for generating HTML based on the email
* template.
*/
public generateHTML(options: GenerateHTMLOptions): string {
return this.env.render(options.name + ".html", options.context);
public async generateHTML(
tenant: Tenant,
template: Template
): Promise<string> {
// Get the environment to render with.
const env = this.getEnvironment(tenant);
// Render the nunjucks template.
return render(env, template);
}
}
+30 -139
View File
@@ -1,139 +1,24 @@
import Queue, { Job } from "bull";
import htmlToText from "html-to-text";
import Joi from "joi";
import { Db } from "mongodb";
import { createTransport } from "nodemailer";
import Queue from "bull";
import now from "performance-now";
import { Config } from "talk-server/config";
import logger from "talk-server/logger";
import Task from "talk-server/queue/Task";
import MailerContent from "talk-server/queue/tasks/mailer/content";
import TenantCache from "talk-server/services/tenant/cache";
import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter";
const JOB_NAME = "mailer";
export interface MailProcessorOptions {
config: Config;
mongo: Db;
tenantCache: TenantCache;
}
export interface MailerData {
message: {
to: string;
subject: string;
html: string;
};
tenantID: string;
}
const MailerDataSchema = Joi.object().keys({
message: Joi.object().keys({
to: Joi.string(),
subject: Joi.string(),
html: Joi.string(),
}),
tenantID: Joi.string(),
});
const createJobProcessor = (options: MailProcessorOptions) => {
const { tenantCache } = options;
// Create the cache adapter that will handle invalidating the email transport
// when the tenant experiences a change.
const cache = new TenantCacheAdapter<ReturnType<typeof createTransport>>(
tenantCache
);
return async (job: Job<MailerData>) => {
const { value, error: err } = Joi.validate(job.data, MailerDataSchema, {
stripUnknown: true,
presence: "required",
abortEarly: false,
});
if (err) {
logger.error(
{
jobID: job.id,
jobName: JOB_NAME,
err,
},
"job data did not match expected schema"
);
return;
}
// Pull the data out of the validated model.
const { message, tenantID } = value;
const log = logger.child({
jobID: job.id,
jobName: JOB_NAME,
tenantID,
});
// Get the referenced tenant so we know who to send it from.
const tenant = await tenantCache.retrieveByID(tenantID);
if (!tenant) {
log.error("referenced tenant was not found");
return;
}
if (!tenant.email.enabled) {
log.error("not sending email, it was disabled");
return;
}
if (!tenant.email.smtpURI) {
log.error("email was enabled but the smtpURI configuration was missing");
return;
}
if (!tenant.email.fromAddress) {
// TODO: possibly have fallback email address?
log.error(
"email was enabled but the fromAddress configuration was missing"
);
return;
}
let transport = cache.get(tenantID);
if (!transport) {
// Create the transport based on the smtp uri.
transport = createTransport(tenant.email.smtpURI);
// Set the transport back into the cache.
cache.set(tenantID, transport);
log.debug("transport was not cached");
} else {
log.debug("transport was cached");
}
log.debug("starting to send the email");
// Send the mail message.
await transport.sendMail({
...message,
// Generate the text content of the message from the HTML.
text: htmlToText.fromString(message.html),
from: tenant.email.fromAddress,
});
log.debug("sent the email");
};
};
import {
createJobProcessor,
JOB_NAME,
MailerData,
MailProcessorOptions,
} from "./processor";
import { Template } from "./templates";
export interface MailerInput {
message: {
to: string;
subject: string;
};
template: {
name: string;
context: object;
};
template: Template;
tenantID: string;
}
@@ -148,13 +33,11 @@ export class MailerQueue {
jobProcessor: createJobProcessor(options),
queue,
});
this.content = new MailerContent(options.config);
this.content = new MailerContent(options);
this.tenantCache = options.tenantCache;
}
public async add({ template, ...rest }: MailerInput) {
const { tenantID } = rest;
public async add({ template, tenantID, message: { to } }: MailerInput) {
const log = logger.child({
jobName: JOB_NAME,
tenantID,
@@ -175,20 +58,28 @@ export class MailerQueue {
return;
}
// Generate the HTML for the email template.
const html = this.content.generateHTML({
...template,
context: {
...template.context,
tenant,
},
});
const startTemplateGenerationTime = now();
let html: string;
try {
// Generate the HTML for the email template.
html = await this.content.generateHTML(tenant, template);
} catch (err) {
log.error({ err }, "could not generate the html");
// TODO: (wyattjoh) maybe throw an error here?
return;
}
// Compute the end time.
const responseTime = Math.round(now() - startTemplateGenerationTime);
log.trace({ responseTime }, "finished template generation");
// Return the job that'll add the email to the queue to be processed later.
return this.task.add({
...rest,
tenantID,
templateName: template.name,
message: {
...rest.message,
to,
html,
},
});
@@ -0,0 +1,274 @@
import { Job } from "bull";
import { DOMLocalization } from "fluent-dom/compat";
import { FluentBundle } from "fluent/compat";
import { minify } from "html-minifier";
import htmlToText from "html-to-text";
import Joi from "joi";
import { JSDOM } from "jsdom";
import { juiceResources } from "juice";
import { camelCase } from "lodash";
import { Db } from "mongodb";
import { createTransport } from "nodemailer";
import now from "performance-now";
import { LanguageCode } from "talk-common/helpers/i18n/locales";
import { Config } from "talk-server/config";
import { InternalError } from "talk-server/errors";
import logger from "talk-server/logger";
import { I18n, translate } from "talk-server/services/i18n";
import TenantCache from "talk-server/services/tenant/cache";
import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter";
export const JOB_NAME = "mailer";
export interface MailProcessorOptions {
config: Config;
mongo: Db;
tenantCache: TenantCache;
i18n: I18n;
}
export interface MailerData {
templateName: string;
message: {
to: string;
html: string;
};
tenantID: string;
}
const MailerDataSchema = Joi.object().keys({
templateName: Joi.string(),
message: Joi.object().keys({
to: Joi.string(),
html: Joi.string(),
}),
tenantID: Joi.string(),
});
interface Message {
from: string;
to: string;
html: string;
text: string;
subject: string;
}
/**
* juiceHTML will juice the HTML to inline the CSS and minify it.
*
* @param input html string to juice
*/
export function juiceHTML(input: string) {
return new Promise<string>((resolve, reject) => {
juiceResources(
input,
{ webResources: { relativeTo: __dirname } },
(err, html) => {
if (err) {
return reject(err);
}
return resolve(
minify(html, {
removeComments: true,
collapseWhitespace: true,
})
);
}
);
});
}
function generateBundleIterator(bundle: FluentBundle) {
return function* generate(resourceIDs: string[]) {
yield bundle;
};
}
function createMessageTranslator(i18n: I18n) {
/**
* translateMessage will translate the message to the specified locale as well
* a juice the contents.
*
* @param templateName the name of the template to base the translations off of
* @param locale the locale to translate the email content into
* @param fromAddress the address that is sending the email (from the Tenant)
* @param data data used to send the message
*/
return async (
templateName: string,
locale: LanguageCode,
fromAddress: string,
data: MailerData
): Promise<Message> => {
// Setup the localization bundles.
const bundle = i18n.getBundle(locale);
const loc = new DOMLocalization([], generateBundleIterator(bundle));
// Translate the HTML fragments.
let dom: JSDOM;
try {
// Parse the rendered template.
dom = new JSDOM(data.message.html, {});
} catch (err) {
logger.error({ err }, "could not parse the HTML for i18n");
throw err;
}
// Translate the bundle.
await loc.translateFragment(dom.window.document);
// TODO: (wyattjoh) strip the i18n attributes from the source.
// Grab the rendered HTML from the dom, and juice them.
if (!dom.window.document.documentElement) {
throw new Error("dom did not have a document element");
}
const translatedHTML = dom.window.document.documentElement.outerHTML;
// Juice the HTML to inline resources.
const html = await juiceHTML(translatedHTML);
// Get the translated subject.
const subject = translate(
bundle,
templateName,
`email-subject-${camelCase(templateName)}`
);
// Generate the text content of the message from the HTML.
const text = htmlToText.fromString(html);
// Prepare the message payload.
return {
from: fromAddress,
to: data.message.to,
html,
text,
subject,
};
};
}
export const createJobProcessor = (options: MailProcessorOptions) => {
const { tenantCache, i18n } = options;
// Create the cache adapter that will handle invalidating the email transport
// when the tenant experiences a change.
const cache = new TenantCacheAdapter<ReturnType<typeof createTransport>>(
tenantCache
);
// Create the message translator function.
const translateMessage = createMessageTranslator(i18n);
return async (job: Job<MailerData>) => {
const { value: data, error: err } = Joi.validate(
job.data,
MailerDataSchema,
{
stripUnknown: true,
presence: "required",
abortEarly: false,
}
);
if (err) {
logger.error(
{
jobID: job.id,
jobName: JOB_NAME,
err,
},
"job data did not match expected schema"
);
return;
}
// Pull the data out of the validated model.
const { tenantID } = data;
const log = logger.child({
jobID: job.id,
jobName: JOB_NAME,
tenantID,
});
// Get the referenced tenant so we know who to send it from.
const tenant = await tenantCache.retrieveByID(tenantID);
if (!tenant) {
log.error("referenced tenant was not found");
return;
}
const { enabled, smtpURI, fromAddress } = tenant.email;
if (!enabled) {
log.error("not sending email, it was disabled");
return;
}
if (!smtpURI) {
log.error("email was enabled but the smtpURI configuration was missing");
return;
}
if (!fromAddress) {
log.error(
"email was enabled but the fromAddress configuration was missing"
);
return;
}
const startTemplateGenerationTime = now();
// Get the message to send.
let message: Message;
try {
message = await translateMessage(
data.templateName,
tenant.locale,
fromAddress,
data
);
} catch (err) {
throw new InternalError(err, "could not translate the message");
}
// Compute the end time.
const responseTime = Math.round(now() - startTemplateGenerationTime);
log.trace({ responseTime }, "finished mail translation");
let transport = cache.get(tenantID);
if (!transport) {
try {
// Create the transport based on the smtp uri.
transport = createTransport(smtpURI);
} catch (err) {
throw new InternalError(err, "could not create email transport");
}
// Set the transport back into the cache.
cache.set(tenantID, transport);
log.debug("transport was not cached");
} else {
log.debug("transport was cached");
}
log.debug("starting to send the email");
const startMessageSendTime = now();
try {
// Send the mail message.
await transport.sendMail(message);
} catch (err) {
throw new InternalError(err, "could not send email");
}
// Compute the end time.
const messageSendResponseTime = Math.round(now() - startMessageSendTime);
log.debug({ responseTime: messageSendResponseTime }, "sent the email");
};
};
@@ -0,0 +1,9 @@
{% extends "layouts/user-notification.html" %}
{% block content %}
Hello {{ context.username }},<br/><br/>
Someone with access to your account has violated our community guidelines.
As a result, your account has been banned. You will no longer be able to
comment, react or report comments. if you think this has been done in error,
please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
{% endblock %}
@@ -0,0 +1,9 @@
{% extends "layouts/user-notification.html" %}
{% block content %}
Hello {{ context.username }},<br/><br/>
To confirm your email address for use with your commenting account at {{ context.organizationName }},
please follow this link: <a data-l10n-name="confirmYourEmail" href="{{ context.confirmURL }}">Click here to confirm your email</a><br/><br/>
If you did not recently create a commenting account with
{{ context.organizationName }}, you can safely ignore this email.
{% endblock %}
@@ -1,14 +1,8 @@
<!doctype html>
<html lang="en">
<head></head>
<body>
<h1>Forgot Password</h1>
<p>We received a request to reset your password on <a href="{{ tenant.organizationURL }}">{{ tenant.organizationName }}</a>.</p>
<p>Please follow the link below to reset your password:</p>
<p><a href="{{ resetURL }}">Click here to reset your password</a></p>
<p><i>If you did not request this change, you can ignore this email.</i></p>
<footer>
<p>Sent by <a href="{{ tenant.organizationURL }}">{{ tenant.organizationName }}</a></p>
</footer>
</body>
</html>
{% extends "layouts/user-notification.html" %}
{% block content %}
Hello {{ context.username }},<br/><br/>
We received a request to reset your password on <a data-l10n-name="organizationName" href="{{ context.organizationURL }}"></a>.<br/><br/>
Please follow this link reset your password: <a data-l10n-name="resetYourPassword" href="{{ context.resetURL }}">Click here to reset your password</a><br/><br/>
<i>If you did not request this, you can ignore this email.</i><br/>
{% endblock %}
@@ -0,0 +1,63 @@
interface Template<T extends string, U extends {}> {
name: T;
context: U;
}
type UserNotificationContext<T extends string, U extends {}> = Template<
T,
U & {
organizationURL: string;
organizationName: string;
}
>;
export type ForgotPasswordTemplate = UserNotificationContext<
"forgot-password",
{
username: string;
resetURL: string;
}
>;
export type BanTemplate = UserNotificationContext<
"ban",
{
username: string;
organizationContactEmail: string;
}
>;
export type SuspendTemplate = UserNotificationContext<
"suspend",
{
username: string;
until: string;
organizationContactEmail: string;
}
>;
export type PasswordChangeTemplate = UserNotificationContext<
"password-change",
{
username: string;
organizationContactEmail: string;
}
>;
export type ConfirmEmailTemplate = UserNotificationContext<
"confirm-email",
{
username: string;
confirmURL: string;
organizationContactEmail: string;
}
>;
type Templates =
| ForgotPasswordTemplate
| BanTemplate
| SuspendTemplate
| PasswordChangeTemplate
| ConfirmEmailTemplate;
export { Templates as Template };
@@ -0,0 +1,14 @@
<!DOCTYPE html>
<html>
<head>
{# Meta tags #}
<meta charset="utf-8" />
{% block meta %}{% endblock %}
{# CSS #}
<link href="assets/main.css" rel="stylesheet">
</head>
<body>
{% block body %}{% endblock %}
</body>
</html>
@@ -0,0 +1,16 @@
{% extends "layouts/base.html" %}
{% block body %}
<table>
<tr>
<td class="content" data-l10n-id="email-notification-template-{{ name }}" data-l10n-args="{{ context | dump }}">
{% block content %}{% endblock %}
</td>
</tr>
<tr>
<td align="center" data-l10n-id="email-notification-footer" data-l10n-args="{{ context | dump }}">
Sent by <a data-l10n-name="organizationLink" href="{{ context.organizationURL }}">{{ context.organizationName }}</a>
</td>
</tr>
</table>
{% endblock %}
@@ -0,0 +1,8 @@
{% extends "layouts/user-notification.html" %}
{% block content %}
Hello {{ context.username }},<br/><br/>
The password on your account has been changed.<br/><br/>
If you did not request this change,
please contact please contact our community team at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
{% endblock %}
@@ -0,0 +1,11 @@
{% extends "layouts/user-notification.html" %}
{% block content %}
Hello {{ context.username }},<br/><br/>
In accordance with {{ context.organizationName }}'s community guidelines, your
account has been temporarily suspended. During the suspension, you will be
unable to comment, flag or engage with fellow commenters. Please rejoin the
conversation {{ context.until }}.<br/><br/>
If you think this has been done in error, please contact our community team
at <a data-l10n-name="organizationContactEmail" href="mailto:{{ context.organizationContactEmail }}">{{ context.organizationContactEmail }}</a>.
{% endblock %}