From 9b0e6ed53bbd2e7ea56b017f4c95d052da79468f Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Wed, 6 Feb 2019 17:53:34 +0000 Subject: [PATCH] [next] MongoDB Indexes (#2142) * feat: added mongo indexing support * fix: fixed typescript issue * chore: better types * fix: revert debug stuff * fix: addressed ts error * feat: added config option to disable auto-indexing * chore: reordered imports * refactor: cleaned up some filepaths --- package-lock.json | 47 ++++------- package.json | 4 +- scripts/generateSchemaTypes.js | 2 +- .../passport/strategies/oidc/index.ts | 6 +- src/core/server/config.ts | 7 ++ .../server/graph/common/scalars/cursor.ts | 2 +- .../server/graph/tenant/loaders/Comments.ts | 2 +- .../server/graph/tenant/resolvers/Comment.ts | 2 +- .../tenant/resolvers/ModerationQueues.ts | 2 +- .../server/graph/tenant/schema/schema.graphql | 30 ------- src/core/server/index.ts | 14 +++- src/core/server/models/action/comment.ts | 17 +++- .../models/action/moderation/comment.ts | 23 +++++- src/core/server/models/comment.ts | 56 ++++++++++++- .../models/{ => helpers}/connection.spec.ts | 0 .../server/models/{ => helpers}/connection.ts | 3 +- src/core/server/models/{ => helpers}/query.ts | 78 ++++++++++++++++++- src/core/server/models/story/counts/empty.ts | 20 +++-- src/core/server/models/story/counts/index.ts | 12 ++- src/core/server/models/story/counts/shared.ts | 74 +++++++++++------- src/core/server/models/story/index.ts | 12 +++ src/core/server/models/tenant.ts | 11 +++ src/core/server/models/user.ts | 41 ++++++++-- src/core/server/services/mongodb/indexes.ts | 45 +++++++++++ src/core/server/services/tenant/index.ts | 4 + src/index.ts | 9 ++- 26 files changed, 397 insertions(+), 126 deletions(-) rename src/core/server/models/{ => helpers}/connection.spec.ts (100%) rename src/core/server/models/{ => helpers}/connection.ts (97%) rename src/core/server/models/{ => helpers}/query.ts (52%) create mode 100644 src/core/server/services/mongodb/indexes.ts diff --git a/package-lock.json b/package-lock.json index 81923c4a7..b8c63d256 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7143,7 +7143,7 @@ }, "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true }, @@ -7951,7 +7951,7 @@ }, "css-color-names": { "version": "0.0.4", - "resolved": "http://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz", + "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz", "integrity": "sha1-gIrcLnnPhHOAabZGyyDsJ762KeA=", "dev": true }, @@ -11378,7 +11378,7 @@ "integrity": "sha1-ETOUSrJHeINHOZVZaIPg05z4hc8=", "dev": true, "requires": { - "intl-pluralrules": "github:projectfluent/IntlPluralRules#module" + "intl-pluralrules": "github:projectfluent/IntlPluralRules#94cb0fa1c23ad943bc5aafef43cea132fa51d68b" } }, "fluent-langneg": { @@ -11672,8 +11672,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "optional": true + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, "aproba": { "version": "1.2.0", @@ -11694,14 +11693,12 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "optional": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -11716,20 +11713,17 @@ "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "optional": true + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "optional": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "optional": true + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" }, "core-util-is": { "version": "1.0.2", @@ -11846,8 +11840,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "optional": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { "version": "1.3.5", @@ -11859,7 +11852,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -11874,7 +11866,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -11882,14 +11873,12 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "optional": true + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, "minipass": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.2.4.tgz", "integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==", - "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -11908,7 +11897,6 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "optional": true, "requires": { "minimist": "0.0.8" } @@ -11989,8 +11977,7 @@ "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "optional": true + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" }, "object-assign": { "version": "4.1.1", @@ -12002,7 +11989,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "optional": true, "requires": { "wrappy": "1" } @@ -12088,8 +12074,7 @@ "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==", - "optional": true + "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, "safer-buffer": { "version": "2.1.2", @@ -12125,7 +12110,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -12145,7 +12129,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -12189,14 +12172,12 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "optional": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "yallist": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz", - "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=", - "optional": true + "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=" } } }, diff --git a/package.json b/package.json index 1727e4375..c7f20782f 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "generate:relay-admin": "ts-node ./scripts/compileRelay --src ./src/core/client/admin --schema tenant", "generate:schema": "node ./scripts/generateSchemaTypes.js", "docz": "docz", - "start": "node dist/index.js", - "start:development": "CONCURRENCY=2 TS_NODE_PROJECT=./src/tsconfig.json node -r ts-node/register -r tsconfig-paths/register ./src/index.ts", + "start": "NODE_ENV=production node dist/index.js", + "start:development": "NODE_ENV=development CONCURRENCY=2 TS_NODE_PROJECT=./src/tsconfig.json node -r ts-node/register -r tsconfig-paths/register ./src/index.ts", "start:webpackDevServer": "ts-node ./scripts/start.ts", "lint": "npm-run-all --parallel lint:* tscheck:*", "lint:server": "tslint --project ./src/tsconfig.json", diff --git a/scripts/generateSchemaTypes.js b/scripts/generateSchemaTypes.js index 11e373843..1d34dfe55 100644 --- a/scripts/generateSchemaTypes.js +++ b/scripts/generateSchemaTypes.js @@ -34,7 +34,7 @@ async function main() { config: { contextType: "TenantContext", importStatements: [ - 'import { Cursor } from "talk-server/models/connection";', + 'import { Cursor } from "talk-server/models/helpers/connection";', 'import TenantContext from "talk-server/graph/tenant/context";', ], customScalarType: { Cursor: "Cursor", Time: "Date" }, diff --git a/src/core/server/app/middleware/passport/strategies/oidc/index.ts b/src/core/server/app/middleware/passport/strategies/oidc/index.ts index 181e5f76a..4d7c9bfd2 100644 --- a/src/core/server/app/middleware/passport/strategies/oidc/index.ts +++ b/src/core/server/app/middleware/passport/strategies/oidc/index.ts @@ -151,7 +151,11 @@ export async function findOrCreateOIDCUser( }; // Try to lookup user given their id provided in the `sub` claim. - let user = await retrieveUserWithProfile(db, tenant.id, profile); + let user = await retrieveUserWithProfile(db, tenant.id, { + // NOTE: (wyattjoh) as the current requirements do not allow multiple OIDC integrations, we are only getting the profile based on the OIDC provider. + type: "oidc", + id: sub, + }); if (!user) { if (!integration.allowRegistration) { // Registration is disabled, so we can't create the user user here. diff --git a/src/core/server/config.ts b/src/core/server/config.ts index 0f3d3720a..7b0160c84 100644 --- a/src/core/server/config.ts +++ b/src/core/server/config.ts @@ -140,6 +140,13 @@ const config = convict({ env: "DISABLE_TENANT_CACHING", arg: "disableTenantCaching", }, + disable_mongodb_autoindexing: { + doc: "Disables the creation of new MongoDB indexes", + format: Boolean, + default: false, + env: "DISABLE_MONGODB_AUTOINDEXING", + arg: "disableMongoDBAutoindexing", + }, }); export type Config = typeof config; diff --git a/src/core/server/graph/common/scalars/cursor.ts b/src/core/server/graph/common/scalars/cursor.ts index e4aeaab74..a5f7eb12f 100644 --- a/src/core/server/graph/common/scalars/cursor.ts +++ b/src/core/server/graph/common/scalars/cursor.ts @@ -2,7 +2,7 @@ import { GraphQLScalarType } from "graphql"; import { Kind } from "graphql/language"; import { DateTime } from "luxon"; -import { Cursor } from "talk-server/models/connection"; +import { Cursor } from "talk-server/models/helpers/connection"; function parseIntegerCursor(value: string): number | null { try { diff --git a/src/core/server/graph/tenant/loaders/Comments.ts b/src/core/server/graph/tenant/loaders/Comments.ts index 105374c1c..fe6b818b5 100644 --- a/src/core/server/graph/tenant/loaders/Comments.ts +++ b/src/core/server/graph/tenant/loaders/Comments.ts @@ -19,7 +19,7 @@ import { retrieveCommentUserConnection, retrieveManyComments, } from "talk-server/models/comment"; -import { Connection } from "talk-server/models/connection"; +import { Connection } from "talk-server/models/helpers/connection"; import { retrieveSharedModerationQueueQueuesCounts } from "talk-server/models/story/counts/shared"; import { SingletonResolver } from "./util"; diff --git a/src/core/server/graph/tenant/resolvers/Comment.ts b/src/core/server/graph/tenant/resolvers/Comment.ts index ad90e647d..3e7594cd2 100644 --- a/src/core/server/graph/tenant/resolvers/Comment.ts +++ b/src/core/server/graph/tenant/resolvers/Comment.ts @@ -7,7 +7,7 @@ import { import { decodeActionCounts } from "talk-server/models/action/comment"; import * as comment from "talk-server/models/comment"; import { getLatestRevision } from "talk-server/models/comment"; -import { createConnection } from "talk-server/models/connection"; +import { createConnection } from "talk-server/models/helpers/connection"; import TenantContext from "../context"; import { getURLWithCommentID } from "./util"; diff --git a/src/core/server/graph/tenant/resolvers/ModerationQueues.ts b/src/core/server/graph/tenant/resolvers/ModerationQueues.ts index f0ac57da4..79eeab1c3 100644 --- a/src/core/server/graph/tenant/resolvers/ModerationQueues.ts +++ b/src/core/server/graph/tenant/resolvers/ModerationQueues.ts @@ -4,7 +4,7 @@ import { RejectCommentPayloadToModerationQueuesResolver, } from "talk-server/graph/tenant/schema/__generated__/types"; import { CommentConnectionInput } from "talk-server/models/comment"; -import { FilterQuery } from "talk-server/models/query"; +import { FilterQuery } from "talk-server/models/helpers/query"; import { CommentModerationCountsPerQueue, Story, diff --git a/src/core/server/graph/tenant/schema/schema.graphql b/src/core/server/graph/tenant/schema/schema.graphql index 9430abf4b..15d2cdedb 100644 --- a/src/core/server/graph/tenant/schema/schema.graphql +++ b/src/core/server/graph/tenant/schema/schema.graphql @@ -1547,36 +1547,6 @@ type Story { createdAt: Time! } -""" -StoryEdge represents a unique Story in a StoryConnection. -""" -type StoryEdge { - """ - node is the Story for this edge. - """ - node: Story! - - """ - - """ - cursor: Cursor! -} - -""" -StoriesConnection represents a subset of a Story list. -""" -type StoriesConnection { - """ - edges are a subset of StoryEdge's. - """ - edges: [StoryEdge!]! - - """ - pageInfo is - """ - pageInfo: PageInfo! -} - ################################################################################ ## CommentsFilterInput ################################################################################ diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 22cca5f7f..40588d819 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -15,6 +15,7 @@ import logger from "talk-server/logger"; import { createQueue, TaskQueue } from "talk-server/queue"; import { createJWTSigningConfig } from "talk-server/services/jwt"; import { createMongoDB } from "talk-server/services/mongodb"; +import { ensureIndexes } from "talk-server/services/mongodb/indexes"; import { AugmentedRedis, createRedisClient } from "talk-server/services/redis"; import TenantCache from "talk-server/services/tenant/cache"; @@ -22,6 +23,10 @@ export interface ServerOptions { config?: Config; } +export interface ServerConnectOptions { + isWorker?: boolean; +} + /** * Server provides an interface to create, start, and manage a Talk Server. */ @@ -74,7 +79,7 @@ class Server { }; } - public async connect() { + public async connect({ isWorker }: ServerConnectOptions = {}) { // Guard against double connecting. if (this.connected) { throw new Error("server has already connected"); @@ -84,6 +89,13 @@ class Server { // Setup MongoDB. this.mongo = await createMongoDB(config); + // If the instance being connected is not a worker process, then create the + // database indexes if it isn't disabled. + if (!isWorker && !this.config.get("disable_mongodb_autoindexing")) { + // Setup the database indexes. + await ensureIndexes(this.mongo); + } + // Setup Redis. this.redis = createRedisClient(config); diff --git a/src/core/server/models/action/comment.ts b/src/core/server/models/action/comment.ts index 52c10b43e..f9cbb8530 100644 --- a/src/core/server/models/action/comment.ts +++ b/src/core/server/models/action/comment.ts @@ -11,7 +11,10 @@ import { GQLCOMMENT_FLAG_REASON, GQLCOMMENT_FLAG_REPORTED_REASON, } from "talk-server/graph/tenant/schema/__generated__/types"; -import { FilterQuery } from "talk-server/models/query"; +import { + createIndexFactory, + FilterQuery, +} from "talk-server/models/helpers/query"; import { TenantResource } from "talk-server/models/tenant"; function collection(db: Db) { @@ -103,6 +106,16 @@ export interface CommentAction extends TenantResource { metadata?: Record; } +export async function createCommentActionIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // { actionType, commentID } + await createIndex({ tenantID: 1, actionType: 1, commentID: 1, userID: 1 }); +} + const ActionSchema = [ // Flags { @@ -245,9 +258,9 @@ export async function retrieveUserAction( ) { return collection(mongo).findOne({ tenantID, + actionType, commentID, userID, - actionType, }); } diff --git a/src/core/server/models/action/moderation/comment.ts b/src/core/server/models/action/moderation/comment.ts index 09afd9c0e..f2b7a2da1 100644 --- a/src/core/server/models/action/moderation/comment.ts +++ b/src/core/server/models/action/moderation/comment.ts @@ -8,8 +8,11 @@ import { ConnectionInput, getPageInfo, nodesToEdges, -} from "talk-server/models/connection"; -import Query from "talk-server/models/query"; +} from "talk-server/models/helpers/connection"; +import Query, { + createConnectionOrderVariants, + createIndexFactory, +} from "talk-server/models/helpers/query"; import { TenantResource } from "talk-server/models/tenant"; function collection(db: Db) { @@ -18,6 +21,22 @@ function collection(db: Db) { ); } +export async function createCommentModerationActionIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + const createVariants = createConnectionOrderVariants< + Readonly + >([{ createdAt: -1 }]); + + // { moderatorID, ...connectionParams } + await createVariants(createIndex, { + moderatorID: 1, + }); +} + /** * CommentModerationAction stores information around a moderation action that * was created for a given Comment Revision. diff --git a/src/core/server/models/comment.ts b/src/core/server/models/comment.ts index 4dad31a5a..a30f4a3bd 100644 --- a/src/core/server/models/comment.ts +++ b/src/core/server/models/comment.ts @@ -19,8 +19,11 @@ import { nodesToEdges, NodeToCursorTransformer, OrderedConnectionInput, -} from "talk-server/models/connection"; -import Query from "talk-server/models/query"; +} from "talk-server/models/helpers/connection"; +import Query, { + createConnectionOrderVariants, + createIndexFactory, +} from "talk-server/models/helpers/query"; import { TenantResource } from "talk-server/models/tenant"; function collection(mongo: Db) { @@ -137,6 +140,54 @@ export interface Comment extends TenantResource { metadata?: Record; } +export async function createCommentIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + const variants = createConnectionOrderVariants>([ + { createdAt: -1 }, + { createdAt: 1 }, + { replyCount: -1, createdAt: -1 }, + { "actionCounts.REACTION": -1, createdAt: -1 }, + ]); + + // Story based Comment Connection pagination. + // { storyID, ...connectionParams } + await variants(createIndex, { + tenantID: 1, + storyID: 1, + status: 1, + }); + + // Story based Comment Connection pagination that are flagged. + // { storyID, ...connectionParams } + await variants(createIndex, { + tenantID: 1, + storyID: 1, + status: 1, + "actionCounts.FLAG": 1, + }); + + // Story + Reply based Comment Connection pagination. + // { storyID, ...connectionParams } + await variants(createIndex, { + tenantID: 1, + storyID: 1, + parentID: 1, + status: 1, + }); + + // Author based Comment Connection pagination. + // { authorID, ...connectionParams } + await variants(createIndex, { + tenantID: 1, + authorID: 1, + status: 1, + }); +} + export type CreateCommentInput = Omit< Comment, | "id" @@ -670,6 +721,7 @@ function applyInputToQuery( input: CommentConnectionInput, query: Query ) { + // NOTE: (wyattjoh) if we ever extend these, ensure that the new order variant is added as an index into the `createCommentConnectionOrderVariants` function. switch (input.orderBy) { case GQLCOMMENT_SORT.CREATED_AT_DESC: query.orderBy({ createdAt: -1 }); diff --git a/src/core/server/models/connection.spec.ts b/src/core/server/models/helpers/connection.spec.ts similarity index 100% rename from src/core/server/models/connection.spec.ts rename to src/core/server/models/helpers/connection.spec.ts diff --git a/src/core/server/models/connection.ts b/src/core/server/models/helpers/connection.ts similarity index 97% rename from src/core/server/models/connection.ts rename to src/core/server/models/helpers/connection.ts index 9a1949e57..e3a02fce9 100644 --- a/src/core/server/models/connection.ts +++ b/src/core/server/models/helpers/connection.ts @@ -1,5 +1,6 @@ import { merge } from "lodash"; -import { FilterQuery } from "./query"; + +import { FilterQuery } from "talk-server/models/helpers/query"; export type Cursor = Date | number | string | null; diff --git a/src/core/server/models/query.ts b/src/core/server/models/helpers/query.ts similarity index 52% rename from src/core/server/models/query.ts rename to src/core/server/models/helpers/query.ts index 6e3200d70..6160720ba 100644 --- a/src/core/server/models/query.ts +++ b/src/core/server/models/helpers/query.ts @@ -1,7 +1,13 @@ import { merge } from "lodash"; -import { Collection, Cursor } from "mongodb"; -import { FilterQuery as MongoFilterQuery } from "mongodb"; -import { Writeable } from "../../common/types"; +import { + Collection, + Cursor, + FilterQuery as MongoFilterQuery, + IndexOptions, +} from "mongodb"; + +import { Writeable } from "talk-common/types"; +import logger from "talk-server/logger"; /** * FilterQuery ensures that given the type T, that the FilterQuery will be a @@ -90,3 +96,69 @@ export default class Query { return cursor; } } + +type IndexType = 1 | -1; + +export type IndexSpecification = { + [P in keyof Writeable>]: IndexType +} & + Record; + +type IndexCreationFunction = ( + indexSpec: IndexSpecification, + indexOptions?: IndexOptions +) => Promise; + +export function createIndexFactory( + collection: Collection +): IndexCreationFunction { + const log = logger.child({ + collectionName: collection.collectionName, + }); + + return async ( + indexSpec: IndexSpecification, + indexOptions: IndexOptions = {} + ) => { + try { + // Try to create the index. + const indexName = await collection.createIndex(indexSpec, indexOptions); + log.debug({ indexName, indexSpec, indexOptions }, "index was created"); + + // Match the interface from the `createIndex` function by returning the + // index name. + return indexName; + } catch (err) { + log.error({ err, indexSpec, indexOptions }, "could not create index"); + + // Rethrow the error here. + throw err; + } + }; +} + +export function createConnectionOrderVariants( + variants: Array> +) { + return async ( + createIndex: IndexCreationFunction, + indexSpec: IndexSpecification + ) => { + /** + * createIndexVariant will create a variant on the specified `indexSpec` that + * will include the new variation. + * + * @param variantSpec the spec that makes this variant different + */ + const createIndexVariant = (variantSpec: IndexSpecification) => + createIndex(merge({}, indexSpec, variantSpec)); + + // Create a raw index without the variants applied. + await createIndex(indexSpec); + + // Create all the variants. + for (const variant of variants) { + await createIndexVariant(variant); + } + }; +} diff --git a/src/core/server/models/story/counts/empty.ts b/src/core/server/models/story/counts/empty.ts index e7826319e..716df7b0a 100644 --- a/src/core/server/models/story/counts/empty.ts +++ b/src/core/server/models/story/counts/empty.ts @@ -1,15 +1,23 @@ import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__/types"; -import { CommentModerationQueueCounts, CommentStatusCounts } from "."; +import { + CommentModerationCountsPerQueue, + CommentModerationQueueCounts, + CommentStatusCounts, +} from "."; + +export function createEmptyCommentModerationCountsPerQueue(): CommentModerationCountsPerQueue { + return { + unmoderated: 0, + reported: 0, + pending: 0, + }; +} export function createEmptyCommentModerationQueueCounts(): CommentModerationQueueCounts { return { total: 0, - queues: { - unmoderated: 0, - reported: 0, - pending: 0, - }, + queues: createEmptyCommentModerationCountsPerQueue(), }; } diff --git a/src/core/server/models/story/counts/index.ts b/src/core/server/models/story/counts/index.ts index 1c17c9bca..a14eeea3b 100644 --- a/src/core/server/models/story/counts/index.ts +++ b/src/core/server/models/story/counts/index.ts @@ -1,17 +1,18 @@ export * from "./empty"; export * from "./shared"; +import { identity, isEmpty, pickBy } from "lodash"; import { Db } from "mongodb"; -import { identity, isEmpty, pickBy } from "lodash"; import { DeepPartial } from "talk-common/types"; import { dotize } from "talk-common/utils/dotize"; import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__/types"; import logger from "talk-server/logger"; import { EncodedCommentActionCounts } from "talk-server/models/action/comment"; +import { createIndexFactory } from "talk-server/models/helpers/query"; +import { retrieveStory, Story } from "talk-server/models/story"; import { AugmentedRedis } from "talk-server/services/redis"; -import { retrieveStory, Story } from ".."; import { createEmptyCommentStatusCounts } from "./empty"; import { updateSharedCommentCounts } from "./shared"; @@ -23,6 +24,13 @@ function collection(db: Db) { return db.collection>("stories"); } +export async function createStoryCountIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // { createdAt } + await createIndex({ tenantID: 1, createdAt: 1 }); +} + // TODO: (wyattjoh) write a test to verify that this set of counts is always in sync with GQLCOMMENT_STATUS. /** diff --git a/src/core/server/models/story/counts/shared.ts b/src/core/server/models/story/counts/shared.ts index f7272e847..418e6c5af 100644 --- a/src/core/server/models/story/counts/shared.ts +++ b/src/core/server/models/story/counts/shared.ts @@ -4,15 +4,16 @@ import ms from "ms"; import logger from "talk-server/logger"; import { EncodedCommentActionCounts } from "talk-server/models/action/comment"; -import { AugmentedPipeline, AugmentedRedis } from "talk-server/services/redis"; - +import { Story } from "talk-server/models/story"; import { CommentModerationCountsPerQueue, CommentStatusCounts, StoryCounts, -} from "."; -import { Story } from ".."; +} from "talk-server/models/story/counts"; +import { AugmentedPipeline, AugmentedRedis } from "talk-server/services/redis"; + import { + createEmptyCommentModerationCountsPerQueue, createEmptyCommentModerationQueueCounts, createEmptyCommentStatusCounts, } from "./empty"; @@ -348,21 +349,28 @@ export async function recalculateSharedCommentCounts( ]); } -/** - * stringObjectToNumber will convert an object that has string keys and string - * values into string keys and number values. - */ -function stringObjectToNumber( - input: Record, - defaultValue: number = 0 -): T { - return Object.entries(input).reduce( - (acc, [key, value]) => ({ - ...acc, - [key]: parseInt(value, 10) || defaultValue, - }), - {} - ) as T; +function fillAndConvertStringToNumber< + T extends { [P in keyof T]?: string } & + { [P in Exclude]?: never }, + U extends { [P in keyof U]: number } +>(input: T, initial: U): U { + const result: U = Object.assign({}, initial); + for (const key in input) { + if (!input.hasOwnProperty(key)) { + continue; + } + + // Pull out the value. + const value: string | undefined = input[key] as any; + if (!value) { + continue; + } + + // I know, not ideal, but... + (result as any)[key] = parseInt(value, 10) || 0; + } + + return result; } /** @@ -378,7 +386,7 @@ export async function retrieveSharedActionCommentCounts( mongo: Db, redis: AugmentedRedis, tenantID: string -) { +): Promise { const key = commentCountsActionKey(tenantID); const freshKey = freshenKey(key); @@ -395,7 +403,10 @@ export async function retrieveSharedActionCommentCounts( return recalculateSharedActionCommentCounts(mongo, redis, tenantID); } - return stringObjectToNumber(actions); + return fillAndConvertStringToNumber( + actions, + {} as EncodedCommentActionCounts + ); } /** @@ -411,13 +422,13 @@ export async function retrieveSharedStatusCommentCounts( mongo: Db, redis: AugmentedRedis, tenantID: string -) { +): Promise { const key = commentCountsStatusKey(tenantID); const freshKey = freshenKey(key); // Get the values, and the freshness key. const [[, statuses], [, fresh]]: [ - [Error | undefined, Record | null], + [Error | undefined, Record | null], [Error | undefined, string | null] ] = await redis .pipeline() @@ -428,7 +439,10 @@ export async function retrieveSharedStatusCommentCounts( return recalculateSharedStatusCommentCounts(mongo, redis, tenantID); } - return stringObjectToNumber(statuses); + return fillAndConvertStringToNumber( + statuses, + createEmptyCommentStatusCounts() + ); } /** @@ -473,13 +487,16 @@ export async function retrieveSharedModerationQueueQueuesCounts( mongo: Db, redis: AugmentedRedis, tenantID: string -) { +): Promise { const key = commentCountsModerationQueueQueuesKey(tenantID); const freshKey = freshenKey(key); // Get the values, and the freshness key. const [[, queues], [, fresh]]: [ - [Error | undefined, Record | null], + [ + Error | undefined, + Record | null + ], [Error | undefined, string | null] ] = await redis .pipeline() @@ -493,7 +510,10 @@ export async function retrieveSharedModerationQueueQueuesCounts( logger.debug({ tenantID }, "comment moderation counts were cached"); - return stringObjectToNumber(queues); + return fillAndConvertStringToNumber( + queues, + createEmptyCommentModerationCountsPerQueue() + ); } export async function updateSharedCommentCounts( diff --git a/src/core/server/models/story/index.ts b/src/core/server/models/story/index.ts index 471f7c320..69b940e8e 100644 --- a/src/core/server/models/story/index.ts +++ b/src/core/server/models/story/index.ts @@ -4,8 +4,10 @@ import uuid from "uuid"; import { Omit } from "talk-common/types"; import { dotize } from "talk-common/utils/dotize"; import { GQLStoryMetadata } from "talk-server/graph/tenant/schema/__generated__/types"; +import { createIndexFactory } from "talk-server/models/helpers/query"; import { ModerationSettings } from "talk-server/models/settings"; import { TenantResource } from "talk-server/models/tenant"; + import { createEmptyCommentModerationQueueCounts, createEmptyCommentStatusCounts, @@ -60,6 +62,16 @@ export interface Story extends TenantResource { createdAt: Date; } +export async function createStoryIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // UNIQUE { url } + await createIndex({ tenantID: 1, url: 1 }, { unique: true }); +} + export interface UpsertStoryInput { id?: string; url: string; diff --git a/src/core/server/models/tenant.ts b/src/core/server/models/tenant.ts index 9cd0e0762..4f7bd2b9e 100644 --- a/src/core/server/models/tenant.ts +++ b/src/core/server/models/tenant.ts @@ -6,6 +6,7 @@ import uuid from "uuid"; import { DeepPartial, Omit, Sub } from "talk-common/types"; import { dotize, DotizeOptions } from "talk-common/utils/dotize"; import { GQLMODERATION_MODE } from "talk-server/graph/tenant/schema/__generated__/types"; +import { createIndexFactory } from "talk-server/models/helpers/query"; import { Settings } from "talk-server/models/settings"; function collection(db: Db) { @@ -42,6 +43,16 @@ export interface Tenant extends Settings { organizationContactEmail: string; } +export async function createTenantIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ id: 1 }, { unique: true }); + + // UNIQUE { domain } + await createIndex({ domain: 1 }, { unique: true }); +} + /** * CreateTenantInput is the set of properties that can be set when a given * Tenant is created. The remainder of the properties are set from defaults and diff --git a/src/core/server/models/user.ts b/src/core/server/models/user.ts index feb6bf5fb..5e98ae864 100644 --- a/src/core/server/models/user.ts +++ b/src/core/server/models/user.ts @@ -4,11 +4,14 @@ import uuid from "uuid"; import { Omit, Sub } from "talk-common/types"; import { GQLUSER_ROLE } from "talk-server/graph/tenant/schema/__generated__/types"; -import { FilterQuery } from "talk-server/models/query"; +import { + createIndexFactory, + FilterQuery, +} from "talk-server/models/helpers/query"; import { TenantResource } from "talk-server/models/tenant"; -function collection(db: Db) { - return db.collection>("users"); +function collection(mongo: Db) { + return mongo.collection>("users"); } export interface LocalProfile { @@ -66,6 +69,34 @@ export interface User extends TenantResource { createdAt: Date; } +export async function createUserIndexes(mongo: Db) { + const createIndex = createIndexFactory(collection(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // UNIQUE - PARTIAL { lowercaseUsername } + await createIndex( + { tenantID: 1, lowercaseUsername: 1 }, + { + unique: true, + partialFilterExpression: { lowercaseUsername: { $exists: true } }, + } + ); + + // UNIQUE - PARTIAL { email } + await createIndex( + { tenantID: 1, email: 1 }, + { unique: true, partialFilterExpression: { email: { $exists: true } } } + ); + + // UNIQUE { profiles.type, profiles.id } + await createIndex( + { tenantID: 1, "profiles.type": 1, "profiles.id": 1 }, + { unique: true } + ); +} + function hashPassword(password: string): Promise { return bcrypt.hash(password, 10); } @@ -171,7 +202,7 @@ const createUpsertUserFilter = (user: Readonly) => { }; export async function retrieveUser(db: Db, tenantID: string, id: string) { - return collection(db).findOne({ id, tenantID }); + return collection(db).findOne({ tenantID, id }); } export async function retrieveManyUsers( @@ -194,7 +225,7 @@ export async function retrieveManyUsers( export async function retrieveUserWithProfile( db: Db, tenantID: string, - profile: Partial + profile: Partial> ) { return collection(db).findOne({ tenantID, diff --git a/src/core/server/services/mongodb/indexes.ts b/src/core/server/services/mongodb/indexes.ts new file mode 100644 index 000000000..5469eaa14 --- /dev/null +++ b/src/core/server/services/mongodb/indexes.ts @@ -0,0 +1,45 @@ +import { Db } from "mongodb"; + +import logger from "talk-server/logger"; +import { createCommentActionIndexes } from "talk-server/models/action/comment"; +import { createCommentModerationActionIndexes } from "talk-server/models/action/moderation/comment"; +import { createCommentIndexes } from "talk-server/models/comment"; +import { + createStoryCountIndexes, + createStoryIndexes, +} from "talk-server/models/story"; +import { createTenantIndexes } from "talk-server/models/tenant"; +import { createUserIndexes } from "talk-server/models/user"; + +type IndexCreationFunction = (mongo: Db) => Promise; + +const indexes: Array<[string, IndexCreationFunction]> = [ + ["users", createUserIndexes], + ["tenants", createTenantIndexes], + ["comments", createCommentIndexes], + ["stories", createStoryIndexes], + ["stories", createStoryCountIndexes], + ["commentActions", createCommentActionIndexes], + ["commentModerationActions", createCommentModerationActionIndexes], +]; + +/** + * ensureIndexes will ensure that all indexes have been created. + * + * @param mongo a MongoDB Database Connection + */ +export async function ensureIndexes(mongo: Db) { + logger.debug( + { indexGroupCount: indexes.length }, + "now ensuring indexes are created" + ); + + // For each of the index functions, call it. + for (const [indexGroup, indexFunction] of indexes) { + logger.debug({ indexGroup }, "ensuring indexes are created"); + await indexFunction(mongo); + logger.debug({ indexGroup }, "indexes have been created"); + } + + logger.debug("all indexes have been created"); +} diff --git a/src/core/server/services/tenant/index.ts b/src/core/server/services/tenant/index.ts index 4e9305780..ed077bddf 100644 --- a/src/core/server/services/tenant/index.ts +++ b/src/core/server/services/tenant/index.ts @@ -50,6 +50,10 @@ export async function install( ); } + // TODO: (wyattjoh) perform any pending migrations. + + // TODO: (wyattjoh) setup database indexes. + // Create the Tenant. const tenant = await createTenant(mongo, input); diff --git a/src/index.ts b/src/index.ts index 12b0de76e..bbbd32e97 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ async function worker(server: Server) { logger.debug("started server worker"); // Connect the server to databases. - await server.connect(); + await server.connect({ isWorker: true }); // Start the server. await server.start(app); @@ -31,6 +31,9 @@ async function worker(server: Server) { // master will start the master process. async function master(server: Server) { + const workerCount = server.config.get("concurrency"); + logger.debug({ workerCount }, "spawning workers to handle traffic"); + try { // Connect the server to databases. await server.connect(); @@ -60,7 +63,7 @@ async function bootstrap() { ); // Connect the server to databases. - await server.connect(); + await server.connect({}); // Process jobs. await server.process(); @@ -68,8 +71,6 @@ async function bootstrap() { // Start the server. await server.start(app); } else { - logger.debug({ workerCount }, "spawning workers to handle traffic"); - // Launch the server start within throng. throng({ workers: workerCount,