({
+ jobName: JOB_NAME,
+ jobProcessor: createJobProcessor(options),
+ queue,
+ });
+ this.content = new MailerContent(options.config);
+ this.tenantCache = options.tenantCache;
+ }
+
+ public async add({ template, ...rest }: MailerInput) {
+ const { tenantID } = rest;
+
+ // All email templates require the tenant in order to insert the footer, so
+ // load it from the tenant cache here.
+ const tenant = await this.tenantCache.retrieveByID(tenantID);
+ if (!tenant) {
+ logger.error(
+ {
+ job_name: JOB_NAME,
+ tenant_id: tenantID,
+ },
+ "referenced tenant was not found"
+ );
+ // TODO: (wyattjoh) maybe throw an error here?
+ return;
+ }
+
+ if (!tenant.email.enabled) {
+ logger.error(
+ {
+ job_name: JOB_NAME,
+ tenant_id: tenantID,
+ },
+ "not adding email, it was disabled"
+ );
+ // TODO: (wyattjoh) maybe throw an error here?
+ return;
+ }
+
+ // Generate the HTML for the email template.
+ const html = this.content.generateHTML({
+ ...template,
+ context: {
+ ...template.context,
+ tenant,
+ },
+ });
+
+ // Return the job that'll add the email to the queue to be processed later.
+ return this.task.add({
+ ...rest,
+ message: {
+ ...rest.message,
+ html,
+ },
+ });
+ }
+}
+
+export const createMailerTask = (
+ queue: Queue.QueueOptions,
+ options: MailProcessorOptions
+) => new Mailer(queue, options);
diff --git a/src/core/server/services/queue/tasks/mailer/templates/forgot-password.html b/src/core/server/services/queue/tasks/mailer/templates/forgot-password.html
new file mode 100644
index 000000000..cd390f5dd
--- /dev/null
+++ b/src/core/server/services/queue/tasks/mailer/templates/forgot-password.html
@@ -0,0 +1,14 @@
+
+
+
+
+ Forgot Password
+ We received a request to reset your password on {{ tenant.organizationName }}.
+ Please follow the link below to reset your password:
+ Click here to reset your password
+ If you did not request this change, you can ignore this email.
+
+
+
diff --git a/src/core/server/services/queue/tasks/scraper/index.ts b/src/core/server/services/queue/tasks/scraper/index.ts
new file mode 100644
index 000000000..17c71e3fb
--- /dev/null
+++ b/src/core/server/services/queue/tasks/scraper/index.ts
@@ -0,0 +1,173 @@
+import Queue, { Job } from "bull";
+import cheerio from "cheerio";
+import authorScraper from "metascraper-author";
+import dateScraper from "metascraper-date";
+import descriptionScraper from "metascraper-description";
+import imageScraper from "metascraper-image";
+import titleScraper from "metascraper-title";
+import { Db } from "mongodb";
+
+import logger from "talk-server/logger";
+import { updateAsset } from "talk-server/models/asset";
+import Task from "talk-server/services/queue/Task";
+import { modifiedScraper } from "./rules/modified";
+import { sectionScraper } from "./rules/section";
+
+const JOB_NAME = "scraper";
+
+export interface ScrapeProcessorOptions {
+ mongo: Db;
+}
+
+export interface ScraperData {
+ assetID: string;
+ assetURL: string;
+ tenantID: string;
+}
+
+const createJobProcessor = (
+ options: ScrapeProcessorOptions,
+ scraper: Scraper
+) => async (job: Job) => {
+ // Pull out the job data.
+ const { assetID: id, assetURL: url, tenantID } = job.data;
+
+ logger.debug(
+ {
+ job_id: job.id,
+ job_name: JOB_NAME,
+ asset_id: id,
+ asset_url: url,
+ tenant_id: tenantID,
+ },
+ "starting to scrap the asset"
+ );
+
+ // Get the metadata from the scraped html.
+ const meta = await scraper.scrape(url);
+ if (!meta) {
+ logger.error(
+ {
+ job_id: job.id,
+ job_name: JOB_NAME,
+ asset_id: id,
+ asset_url: url,
+ tenant_id: tenantID,
+ },
+ "asset at specified url not found, can not scrape"
+ );
+ return;
+ }
+
+ // Update the Asset with the scraped details.
+ const asset = await updateAsset(options.mongo, tenantID, id, {
+ title: meta.title || undefined,
+ description: meta.description || undefined,
+ image: meta.image ? meta.image : undefined,
+ author: meta.author || undefined,
+ publication_date: meta.date ? new Date(meta.date) : undefined,
+ modified_date: meta.modified ? new Date(meta.modified) : undefined,
+ section: meta.section || undefined,
+ scraped: new Date(),
+ });
+ if (!asset) {
+ logger.error(
+ {
+ job_id: job.id,
+ job_name: JOB_NAME,
+ asset_id: id,
+ asset_url: url,
+ tenant_id: tenantID,
+ },
+ "asset at specified id not found, can not update with metadata"
+ );
+ return;
+ }
+
+ logger.debug(
+ {
+ job_id: job.id,
+ job_name: JOB_NAME,
+ asset_id: asset.id,
+ asset_url: url,
+ tenant_id: tenantID,
+ },
+ "scraped the asset"
+ );
+};
+
+export type Rule = Record<
+ string,
+ Array<
+ (options: { htmlDom: CheerioSelector; url: string }) => string | undefined
+ >
+>;
+
+class Scraper {
+ private rules: Rule[];
+
+ constructor(rules: Rule[]) {
+ this.rules = rules;
+ }
+
+ public async scrape(url: string) {
+ // Grab the page HTML.
+
+ // TODO: investigate adding scraping proxy support.
+ const res = await fetch(url, {});
+ if (res.status !== 200) {
+ return;
+ }
+
+ const html = await res.text();
+
+ // Load the DOM.
+ const htmlDom = cheerio.load(html);
+
+ // Gather the results by evaluating each of the rules.
+ const metadata: Record = {};
+
+ for (const rule of this.rules) {
+ for (const property in rule) {
+ if (!rule.hasOwnProperty(property)) {
+ continue;
+ }
+
+ // Proceed through each of the properties and try to find the mapped
+ // properties.
+ for (const getter of rule[property]) {
+ const value = getter({ htmlDom, url });
+ if (value && value.length > 0) {
+ metadata[property] = value;
+
+ break;
+ }
+ }
+ }
+ }
+
+ return metadata;
+ }
+}
+
+export function createScraperTask(
+ queue: Queue.QueueOptions,
+ options: ScrapeProcessorOptions
+) {
+ // Create the scraper object.
+ const scraper = new Scraper([
+ authorScraper(),
+ dateScraper(),
+ descriptionScraper(),
+ imageScraper(),
+ titleScraper(),
+ modifiedScraper(),
+ sectionScraper(),
+ ]);
+
+ return new Task({
+ jobName: JOB_NAME,
+ jobProcessor: createJobProcessor(options, scraper),
+ queue,
+ });
+}
diff --git a/src/core/server/services/queue/tasks/scraper/rules/modified.ts b/src/core/server/services/queue/tasks/scraper/rules/modified.ts
new file mode 100644
index 000000000..8b7ce6232
--- /dev/null
+++ b/src/core/server/services/queue/tasks/scraper/rules/modified.ts
@@ -0,0 +1,7 @@
+import { Rules } from "metascraper";
+
+export const modifiedScraper = (): Rules => ({
+ modified: [
+ ({ htmlDom: $ }) => $('meta[property="article:modified"]').attr("content"),
+ ],
+});
diff --git a/src/core/server/services/queue/tasks/scraper/rules/section.ts b/src/core/server/services/queue/tasks/scraper/rules/section.ts
new file mode 100644
index 000000000..957560920
--- /dev/null
+++ b/src/core/server/services/queue/tasks/scraper/rules/section.ts
@@ -0,0 +1,7 @@
+import { Rules } from "metascraper";
+
+export const sectionScraper = (): Rules => ({
+ section: [
+ ({ htmlDom: $ }) => $('meta[property="article:section"]').attr("content"),
+ ],
+});
diff --git a/src/core/server/services/redis/index.ts b/src/core/server/services/redis/index.ts
index 8ec7fbb65..78d8e4d0d 100644
--- a/src/core/server/services/redis/index.ts
+++ b/src/core/server/services/redis/index.ts
@@ -6,6 +6,6 @@ import { Config } from "talk-common/config";
*
* @param config application configuration.
*/
-export async function createRedisClient(config: Config): Promise {
+export function createRedisClient(config: Config): Redis {
return new RedisClient(config.get("redis"), {});
}
diff --git a/src/core/server/services/tenant/cache/adapter.ts b/src/core/server/services/tenant/cache/adapter.ts
index feb6269a9..ac2f0b207 100644
--- a/src/core/server/services/tenant/cache/adapter.ts
+++ b/src/core/server/services/tenant/cache/adapter.ts
@@ -1,4 +1,3 @@
-import { Config } from "talk-common/config";
import TenantCache from "talk-server/services/tenant/cache";
export type DeconstructionFn = (tenantID: string, value: T) => Promise;
@@ -12,26 +11,23 @@ export type DeconstructionFn = (tenantID: string, value: T) => Promise;
export class TenantCacheAdapter {
private cache = new Map();
private tenantCache: TenantCache;
- private isCachingEnabled: boolean;
private unsubscribeFn?: () => void;
private deconstructionFn?: DeconstructionFn;
constructor(
tenantCache: TenantCache,
- config: Config,
deconstructionFn?: DeconstructionFn
) {
this.tenantCache = tenantCache;
- this.isCachingEnabled = !config.get("disable_tenant_caching");
this.deconstructionFn = deconstructionFn;
+
+ // Subscribe to updates immediately.
+ this.subscribe();
}
public subscribe() {
- if (this.isCachingEnabled) {
- // Unsubscribe from updates if we
- this.unsubscribe();
-
+ if (this.tenantCache.cachingEnabled && !this.unsubscribeFn) {
this.unsubscribeFn = this.tenantCache.subscribe(async tenant => {
// Get the current set value for the item in the cache.
const value = this.get(tenant.id);
@@ -58,7 +54,7 @@ export class TenantCacheAdapter {
* This will disconnect the map/cache from getting updates.
*/
public unsubscribe() {
- if (this.isCachingEnabled && this.unsubscribeFn) {
+ if (this.tenantCache.cachingEnabled && this.unsubscribeFn) {
this.unsubscribeFn();
}
}
@@ -69,7 +65,7 @@ export class TenantCacheAdapter {
* @param tenantID the tenantID for the cached item
*/
public get(tenantID: string): T | undefined {
- if (this.isCachingEnabled) {
+ if (this.tenantCache.cachingEnabled) {
return this.cache.get(tenantID);
}
@@ -82,11 +78,9 @@ export class TenantCacheAdapter {
* @param tenantID the tenantID for the cached item
* @param value the value to set in the map (if caching is enabled)
*/
- public set(tenantID: string, value: T): TenantCacheAdapter {
- if (this.isCachingEnabled) {
+ public set(tenantID: string, value: T) {
+ if (this.tenantCache.cachingEnabled) {
this.cache.set(tenantID, value);
}
-
- return this;
}
}
diff --git a/src/core/server/services/tenant/cache/index.ts b/src/core/server/services/tenant/cache/index.ts
index 9b14ee35a..d7a41ca0f 100644
--- a/src/core/server/services/tenant/cache/index.ts
+++ b/src/core/server/services/tenant/cache/index.ts
@@ -47,7 +47,11 @@ export default class TenantCache {
private mongo: Db;
private emitter = new EventEmitter();
- private cachingEnabled: boolean;
+
+ /**
+ * cachingEnabled is true when tenant caching has been enabled.
+ */
+ public cachingEnabled: boolean;
constructor(mongo: Db, subscriber: Redis, config: Config) {
this.cachingEnabled = !config.get("disable_tenant_caching");
@@ -124,7 +128,7 @@ export default class TenantCache {
this.tenantsByDomain.prime(tenant.domain, tenant);
});
- logger.debug({ tenants: tenants.length }, "primed tenants");
+ logger.debug({ tenants: tenants.length }, "primed all tenants");
}
/**
diff --git a/src/core/server/services/tenant/index.ts b/src/core/server/services/tenant/index.ts
index 4107e7051..b1403420d 100644
--- a/src/core/server/services/tenant/index.ts
+++ b/src/core/server/services/tenant/index.ts
@@ -1,15 +1,16 @@
import { Redis } from "ioredis";
import { Db } from "mongodb";
+import { GQLSettingsInput } from "talk-server/graph/tenant/schema/__generated__/types";
import {
Tenant,
updateTenant,
- UpdateTenantInput,
+ // UpdateTenantInput,
} from "talk-server/models/tenant";
import TenantCache from "./cache";
-export type UpdateTenant = UpdateTenantInput;
+export type UpdateTenant = GQLSettingsInput;
export async function update(
db: Db,
diff --git a/src/types/dotize.d.ts b/src/types/dotize.d.ts
deleted file mode 100644
index 0b6e1c5f1..000000000
--- a/src/types/dotize.d.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-declare module "dotize" {
- export function convert(obj: Object): Record
-}
diff --git a/src/types/metascraper.d.ts b/src/types/metascraper.d.ts
new file mode 100644
index 000000000..2a327e8c6
--- /dev/null
+++ b/src/types/metascraper.d.ts
@@ -0,0 +1,42 @@
+declare module "metascraper" {
+ export interface Scraper {
+ (
+ options: {
+ url: string;
+ html: string;
+ }
+ ): Promise>;
+ }
+
+ export type Rules = Record<
+ string,
+ Array<(options: { htmlDom: CheerioSelector }) => string | undefined>
+ >;
+
+ export function load(rules: Rules[]): Scraper;
+}
+
+declare module "metascraper-author" {
+ import { Rules } from "metascraper";
+ export default function(): Rules;
+}
+
+declare module "metascraper-date" {
+ import { Rules } from "metascraper";
+ export default function(): Rules;
+}
+
+declare module "metascraper-description" {
+ import { Rules } from "metascraper";
+ export default function(): Rules;
+}
+
+declare module "metascraper-image" {
+ import { Rules } from "metascraper";
+ export default function(): Rules;
+}
+
+declare module "metascraper-title" {
+ import { Rules } from "metascraper";
+ export default function(): Rules;
+}