diff --git a/README.md b/README.md index 2416e789f..5bf09d67a 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ If you don't already have these databases running, you can execute the following assuming you have Docker installed on your local machine: ```bash -docker run -d -p 27017:27017 --restart always --name mongo mongo:3.6 +docker run -d -p 27017:27017 --restart always --name mongo mongo:4.2 docker run -d -p 6379:6379 --restart always --name redis redis:3.2 ``` @@ -624,12 +624,6 @@ the variables in a `.env` file in the root of the project in a simple - `DISABLE_TENANT_CACHING` - When `true`, all tenants will be loaded from the database when needed rather than keeping a in-memory copy in sync via published events on Redis. (Default `false`) -- `DISABLE_MONGODB_AUTOINDEXING` - When `true`, Coral will not perform indexing - operations when it starts up. This can be desired when you've already - installed Coral on the target MongoDB, but want to improve start performance. - **You should not use this parameter unless you know what you're doing! Upgrades - may introduce additional indexes that the application relies on.** - (Default `false`) - `LOCALE` - Specify the default locale to use for all requests without a locale specified. (Default `en-US`) - `ENABLE_GRAPHIQL` - When `true`, it will enable the GraphiQL interface at `/graphiql`. **(🚨 Note 🚨) we do not recommend using this in production environments as it disables many safety features used by the application**. (Default `false`) diff --git a/package.json b/package.json index e5d13643a..ec314e6c2 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "build:development": "NODE_ENV=development npm-run-all generate --parallel lint:client build:client build:server", "build:client": "ts-node --transpile-only ./scripts/build.ts", "build:server": "gulp server", + "migration:create": "ts-node --transpile-only ./scripts/migration/create.ts", "doctoc": "doctoc --title='## Table of Contents' --github README.md", "generate": "npm-run-all generate:css-types generate:schema generate:relay", "generate-persist": "npm-run-all generate:css-types generate:schema generate:relay-persist", diff --git a/scripts/migration/create.ts b/scripts/migration/create.ts new file mode 100644 index 000000000..a27fe43f5 --- /dev/null +++ b/scripts/migration/create.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env ts-node + +/** + * This script can be invoked via: + * + * npm run migration:create + * + * To create new database migrations. + */ + +// tslint:disable: no-console + +import fs from "fs-extra"; +import lodash from "lodash"; +import path from "path"; + +const templateFilePath = path.resolve( + path.join( + __dirname, + "../../src/core/server/services/migrate/migration_sample.ts" + ) +); + +const argv = process.argv.slice(2); +if (argv.length !== 1) { + console.error("usage: npm run migration:create "); + process.exit(1); +} + +// Get the name of the new migration. +const name = lodash.snakeCase(argv[0]); + +// Get the version of the new migration. +const version = Date.now(); + +// Get the filePath of the new migration. +const filePath = path.resolve( + path.join( + __dirname, + `../../src/core/server/services/migrate/migrations/${version}_${name}.ts` + ) +); + +if (fs.existsSync(filePath)) { + console.error(`migration already exists at: ${filePath}`); + process.exit(1); +} + +// Write the template out to the file. +fs.copyFileSync(templateFilePath, filePath); + +console.log(`created new migration at: ${filePath}`); diff --git a/src/core/client/admin/components/UserHistoryDrawer/UserDrawerAccountHistory.tsx b/src/core/client/admin/components/UserHistoryDrawer/UserDrawerAccountHistory.tsx index a3520dcf8..b305d7488 100644 --- a/src/core/client/admin/components/UserHistoryDrawer/UserDrawerAccountHistory.tsx +++ b/src/core/client/admin/components/UserHistoryDrawer/UserDrawerAccountHistory.tsx @@ -107,20 +107,17 @@ const UserDrawerAccountHistory: FunctionComponent = ({ user }) => { }); }); - // FIXME: (wyattjoh) once migration has been performed, remove check - if (user.status.premod) { - // Merge in all the premod history items. - user.status.premod.history.forEach(record => { - history.push({ - kind: "premod", - action: { - action: record.active ? "created" : "removed", - }, - date: new Date(record.createdAt), - takenBy: record.createdBy ? record.createdBy.username : system, - }); + // Merge in all the premod history items. + user.status.premod.history.forEach(record => { + history.push({ + kind: "premod", + action: { + action: record.active ? "created" : "removed", + }, + date: new Date(record.createdAt), + takenBy: record.createdBy ? record.createdBy.username : system, }); - } + }); user.status.username.history.forEach((record, i) => { history.push({ diff --git a/src/core/client/admin/components/UserStatus/UserStatusChangeContainer.tsx b/src/core/client/admin/components/UserStatus/UserStatusChangeContainer.tsx index 1dea6dd91..e72ee13cf 100644 --- a/src/core/client/admin/components/UserStatus/UserStatusChangeContainer.tsx +++ b/src/core/client/admin/components/UserStatus/UserStatusChangeContainer.tsx @@ -66,8 +66,7 @@ const UserStatusChangeContainer: FunctionComponent = props => { }, [user, removeUserSuspension]); const handlePremod = useCallback(() => { - // FIXME: (wyattjoh) once migration has been performed, remove check - if (user.status.premod && user.status.premod.active) { + if (user.status.premod.active) { return; } setShowPremod(true); @@ -83,8 +82,7 @@ const UserStatusChangeContainer: FunctionComponent = props => { }, [setShowPremod]); const handleRemovePremod = useCallback(() => { - // FIXME: (wyattjoh) once migration has been performed, remove check - if (!user.status.premod || !user.status.premod.active) { + if (!user.status.premod.active) { return; } removeUserPremod({ userID: user.id }); @@ -138,8 +136,7 @@ const UserStatusChangeContainer: FunctionComponent = props => { onRemovePremod={handleRemovePremod} banned={user.status.ban.active} suspended={user.status.suspension.active} - // FIXME: (wyattjoh) once migration has been performed, remove check - premod={Boolean(user.status.premod && user.status.premod.active)} + premod={user.status.premod.active} fullWidth={fullWidth} > diff --git a/src/core/server/app/handlers/api/install.ts b/src/core/server/app/handlers/api/install.ts index d4f9b1a35..b38c10376 100644 --- a/src/core/server/app/handlers/api/install.ts +++ b/src/core/server/app/handlers/api/install.ts @@ -54,7 +54,7 @@ const TenantInstallBodySchema = Joi.object().keys({ export type TenantInstallHandlerOptions = Pick< AppOptions, - "redis" | "mongo" | "config" | "mailerQueue" | "i18n" + "redis" | "mongo" | "config" | "mailerQueue" | "i18n" | "migrationManager" >; export const installHandler = ({ @@ -62,6 +62,7 @@ export const installHandler = ({ redis, config, i18n, + migrationManager, }: TenantInstallHandlerOptions): RequestHandler => async (req, res, next) => { try { if (!req.coral) { @@ -108,6 +109,9 @@ export const installHandler = ({ req.coral.now ); + // Execute pending migrations to get everything installed. + await migrationManager.executePendingMigrations(mongo); + // Pull the user details out of the input for the user. const { email, username, password } = userInput; diff --git a/src/core/server/app/index.ts b/src/core/server/app/index.ts index a291f8db9..2b13b051a 100644 --- a/src/core/server/app/index.ts +++ b/src/core/server/app/index.ts @@ -19,6 +19,7 @@ import { ScraperQueue } from "coral-server/queue/tasks/scraper"; import { I18n } from "coral-server/services/i18n"; import { JWTSigningConfig } from "coral-server/services/jwt"; import { Metrics } from "coral-server/services/metrics"; +import { MigrationManager } from "coral-server/services/migrate"; import { PersistedQueryCache } from "coral-server/services/queries"; import { AugmentedRedis } from "coral-server/services/redis"; import TenantCache from "coral-server/services/tenant/cache"; @@ -46,6 +47,7 @@ export interface AppOptions { scraperQueue: ScraperQueue; signingConfig: JWTSigningConfig; tenantCache: TenantCache; + migrationManager: MigrationManager; } /** diff --git a/src/core/server/config.ts b/src/core/server/config.ts index e9fc9076b..bb702739f 100644 --- a/src/core/server/config.ts +++ b/src/core/server/config.ts @@ -204,13 +204,6 @@ const config = convict({ env: "DISABLE_LIVE_UPDATES", arg: "disableLiveUpdates", }, - disable_mongodb_autoindexing: { - doc: "Disables the creation of new MongoDB indexes", - format: Boolean, - default: false, - env: "DISABLE_MONGODB_AUTOINDEXING", - arg: "disableMongoDBAutoindexing", - }, disable_client_routes: { doc: "Disables mounting of client routes for developing with Webpack Dev Server", diff --git a/src/core/server/graph/tenant/resolvers/UserStatus.ts b/src/core/server/graph/tenant/resolvers/UserStatus.ts index 568d24a32..72b9ce217 100644 --- a/src/core/server/graph/tenant/resolvers/UserStatus.ts +++ b/src/core/server/graph/tenant/resolvers/UserStatus.ts @@ -31,8 +31,7 @@ export const UserStatus: Required< } // If they are set to mandatory premod, then mark it. - // FIXME: (wyattjoh) once migration has been performed, remove check - if (consolidatedStatus.premod && consolidatedStatus.premod.active) { + if (consolidatedStatus.premod.active) { statuses.push(GQLUSER_STATUS.PREMOD); } @@ -55,17 +54,8 @@ export const UserStatus: Required< ...user.consolidateUserSuspensionStatus(suspension), userID, }), - // FIXME: (wyattjoh) once migration has been performed, return PremodStatusInput only - premod: ({ premod, userID }): PremodStatusInput | null => { - const status = user.consolidateUserPremodStatus(premod); - // FIXME: (wyattjoh) once migration has been performed, remove check - if (!status) { - return null; - } - - return { - ...status, - userID, - }; - }, + premod: ({ premod, userID }): PremodStatusInput => ({ + ...user.consolidateUserPremodStatus(premod), + userID, + }), }; diff --git a/src/core/server/graph/tenant/schema/schema.graphql b/src/core/server/graph/tenant/schema/schema.graphql index 8f6c089f0..7030eced4 100644 --- a/src/core/server/graph/tenant/schema/schema.graphql +++ b/src/core/server/graph/tenant/schema/schema.graphql @@ -1541,10 +1541,8 @@ type UserStatus { """ premod stores the user premod status as well as the history of changes. - - FIXME: (wyattjoh) once migration has been performed, make non-nullable """ - premod: PremodStatus @auth(roles: [ADMIN, MODERATOR]) + premod: PremodStatus! @auth(roles: [ADMIN, MODERATOR]) } """ diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 71d06d547..7cbef303b 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -27,8 +27,8 @@ import { JWTSigningConfig, } from "coral-server/services/jwt"; import { createMetrics } from "coral-server/services/metrics"; +import { MigrationManager } from "coral-server/services/migrate"; import { createMongoDB } from "coral-server/services/mongodb"; -import { ensureIndexes } from "coral-server/services/mongodb/indexes"; import { PersistedQueryCache } from "coral-server/services/queries"; import { AugmentedRedis, @@ -36,6 +36,7 @@ import { createRedisClient, } from "coral-server/services/redis"; import TenantCache from "coral-server/services/tenant/cache"; +import { isInstalled } from "./services/tenant"; export interface ServerOptions { /** @@ -103,6 +104,9 @@ class Server { // server to handle persisted queries. private persistedQueryCache: PersistedQueryCache; + // migrationManager is the manager for performing migrations on Coral. + private migrationManager: MigrationManager; + constructor(options: ServerOptions) { this.parentApp = express(); @@ -153,6 +157,9 @@ class Server { config ); + // Create the migration manager. + this.migrationManager = new MigrationManager(this.tenantCache); + // Load and upsert the persisted queries. this.persistedQueryCache = new PersistedQueryCache({ mongo: this.mongo }); @@ -188,13 +195,11 @@ class Server { } this.processing = true; - // Create the database indexes if it isn't disabled. - if (!this.config.get("disable_mongodb_autoindexing")) { - // Setup the database indexes. - logger.info("mongodb autoindexing is enabled, starting indexing"); - await ensureIndexes(this.mongo); + // Run migrations if there is already a Tenant installed. + if (await isInstalled(this.tenantCache)) { + await this.migrationManager.executePendingMigrations(this.mongo); } else { - logger.warn("mongodb autoindexing is disabled, skipping indexing"); + logger.info("no tenants are installed, skipping running migrations"); } // Prime the queries in the database. @@ -310,6 +315,7 @@ class Server { persistedQueriesRequired: this.config.get("env") === "production" && !this.config.get("enable_graphiql"), + migrationManager: this.migrationManager, }; // Only enable the metrics server if concurrency is set to 1. diff --git a/src/core/server/logger/serializers.ts b/src/core/server/logger/serializers.ts index 153803f46..d2781dc18 100644 --- a/src/core/server/logger/serializers.ts +++ b/src/core/server/logger/serializers.ts @@ -3,13 +3,14 @@ import { GraphQLError } from "graphql"; import StackUtils from "stack-utils"; import { CoralError, CoralErrorContext } from "coral-server/errors"; +import VError from "verror"; interface SerializedError { id?: string; message: string; name: string; stack?: string; - context?: CoralErrorContext; + context?: CoralErrorContext | Record; originalError?: SerializedError; } @@ -39,6 +40,8 @@ const errSerializer = (err: Error) => { if (cause) { obj.originalError = errSerializer(cause); } + } else if (err instanceof VError) { + obj.context = VError.info(err); } return obj; diff --git a/src/core/server/models/action/comment.ts b/src/core/server/models/action/comment.ts index baa22e84f..b2659bf8b 100644 --- a/src/core/server/models/action/comment.ts +++ b/src/core/server/models/action/comment.ts @@ -17,16 +17,12 @@ import logger from "coral-server/logger"; import { Connection, ConnectionInput, - createCollection, - createConnectionOrderVariants, - createIndexFactory, FilterQuery, Query, resolveConnection, } from "coral-server/models/helpers"; import { TenantResource } from "coral-server/models/tenant"; - -const collection = createCollection("commentActions"); +import { commentActions as collection } from "coral-server/services/mongodb/collections"; export enum ACTION_TYPE { /** @@ -134,32 +130,6 @@ export interface CommentAction extends TenantResource { metadata?: Record; } -export async function createCommentActionIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // { actionType, commentID, userID } - await createIndex( - { tenantID: 1, actionType: 1, commentID: 1, userID: 1 }, - { background: true } - ); - - const variants = createConnectionOrderVariants>( - [{ createdAt: -1 }], - { background: true } - ); - - // Connection pagination. - // { ...connectionParams } - await variants(createIndex, { - tenantID: 1, - actionType: 1, - commentID: 1, - }); -} - const ActionSchema = [ // Flags { diff --git a/src/core/server/models/action/moderation/comment.ts b/src/core/server/models/action/moderation/comment.ts index 67641cc36..776ae0749 100644 --- a/src/core/server/models/action/moderation/comment.ts +++ b/src/core/server/models/action/moderation/comment.ts @@ -6,33 +6,11 @@ import { GQLCOMMENT_STATUS } from "coral-server/graph/tenant/schema/__generated_ import { Connection, ConnectionInput, - createCollection, - createConnectionOrderVariants, - createIndexFactory, Query, resolveConnection, } from "coral-server/models/helpers"; import { TenantResource } from "coral-server/models/tenant"; - -const collection = createCollection( - "commentModerationActions" -); - -export async function createCommentModerationActionIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - const createVariants = createConnectionOrderVariants< - Readonly - >([{ createdAt: -1 }]); - - // { moderatorID, ...connectionParams } - await createVariants(createIndex, { - moderatorID: 1, - }); -} +import { commentModerationActions as collection } from "coral-server/services/mongodb/collections"; /** * CommentModerationAction stores information around a moderation action that diff --git a/src/core/server/models/comment/comment.ts b/src/core/server/models/comment/comment.ts index 62a798e3c..781a949a0 100644 --- a/src/core/server/models/comment/comment.ts +++ b/src/core/server/models/comment/comment.ts @@ -19,10 +19,7 @@ import { } from "coral-server/models/action/comment"; import { Connection, - createCollection, createConnection, - createConnectionOrderVariants, - createIndexFactory, doesNotContainNull, FilterQuery, nodesToEdges, @@ -32,6 +29,7 @@ import { resolveConnection, } from "coral-server/models/helpers"; import { TenantResource } from "coral-server/models/tenant"; +import { comments as collection } from "coral-server/services/mongodb/collections"; import { PUBLISHED_STATUSES } from "./constants"; import { @@ -42,8 +40,6 @@ import { import { Revision } from "./revision"; import { CommentTag } from "./tag"; -const collection = createCollection("comments"); - /** * Comment's are created by User's on Stories. Each Comment contains a body, and * can be moderated by another Moderator or Admin User. @@ -128,84 +124,6 @@ export interface Comment extends TenantResource { deletedAt?: Date; } -export async function createCommentIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // Facility for counting the tags against a story. - await createIndex( - { - tenantID: 1, - storyID: 1, - "tags.type": 1, - status: 1, - }, - { - background: true, - partialFilterExpression: { - "tags.type": { $exists: true }, - }, - } - ); - - const variants = createConnectionOrderVariants>([ - { createdAt: -1 }, - { createdAt: 1 }, - { childCount: -1, createdAt: -1 }, - { "actionCounts.REACTION": -1, createdAt: -1 }, - ]); - - // Story based Comment Connection pagination. - // { storyID, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - storyID: 1, - status: 1, - }); - - // Moderation based Comment Connection pagination. - // { storyID, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - status: 1, - }); - - // Story based Comment Connection pagination that are flagged. - // { storyID, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - storyID: 1, - status: 1, - "actionCounts.FLAG": 1, - }); - - // Story + Reply based Comment Connection pagination. - // { storyID, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - storyID: 1, - parentID: 1, - status: 1, - }); - - // Author based Comment Connection pagination. - // { authorID, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - authorID: 1, - status: 1, - }); - - // Tag based Comment Connection pagination. - // { tags.type, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - "tags.type": 1, - }); -} - export type CreateCommentInput = Omit< Comment, | "id" diff --git a/src/core/server/models/helpers/index.ts b/src/core/server/models/helpers/index.ts index 129528d07..06c3e7441 100644 --- a/src/core/server/models/helpers/index.ts +++ b/src/core/server/models/helpers/index.ts @@ -1,5 +1,4 @@ export * from "./collection"; export * from "./connection"; -export * from "./indexing"; export { default as Query } from "./query"; export * from "./query"; diff --git a/src/core/server/models/invite.ts b/src/core/server/models/invite.ts index cfa61c73f..b05a15856 100644 --- a/src/core/server/models/invite.ts +++ b/src/core/server/models/invite.ts @@ -3,23 +3,8 @@ import uuid from "uuid"; import { Omit, Sub } from "coral-common/types"; import { GQLUSER_ROLE } from "coral-server/graph/tenant/schema/__generated__/types"; -import { - createCollection, - createIndexFactory, -} from "coral-server/models/helpers"; import { TenantResource } from "coral-server/models/tenant"; - -const collection = createCollection>("invites"); - -export async function createInviteIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE { email } - await createIndex({ tenantID: 1, email: 1 }, { unique: true }); -} +import { invites as collection } from "coral-server/services/mongodb/collections"; export interface Invite extends TenantResource { readonly id: string; diff --git a/src/core/server/models/migration/index.ts b/src/core/server/models/migration/index.ts new file mode 100644 index 000000000..585fc62a9 --- /dev/null +++ b/src/core/server/models/migration/index.ts @@ -0,0 +1 @@ +export * from "./migration"; diff --git a/src/core/server/models/migration/migration.ts b/src/core/server/models/migration/migration.ts new file mode 100644 index 000000000..b6d41ad1a --- /dev/null +++ b/src/core/server/models/migration/migration.ts @@ -0,0 +1,96 @@ +import { Db } from "mongodb"; + +import { migrations as collection } from "coral-server/services/mongodb/collections"; + +export enum MIGRATION_STATE { + STARTED = "STARTED", + FAILED = "FAILED", + FINISHED = "FINISHED", +} + +export interface MigrationRecord { + id: number; + state: MIGRATION_STATE; + clientID?: string; + createdAt: Date; + updatedAt?: Date; +} + +export async function startMigration( + mongo: Db, + id: number, + clientID: string, + now = new Date() +) { + const result = await collection(mongo).findOneAndUpdate( + { id }, + { + $setOnInsert: { + id, + clientID, + state: MIGRATION_STATE.STARTED, + createdAt: now, + }, + }, + { + // False to return the updated document instead of the original document. + returnOriginal: false, + upsert: true, + } + ); + if (!result.value) { + throw new Error("an unexpected error occurred"); + } + + return result.value; +} + +/** + * updateMigrationState will update the state of a migration record to reflect + * the new state as well as un-setting the client ID from the records. + * + * @param mongo the database to interact on + * @param id the migration version to update + * @param state the state to switch the record to + * @param now the current date + */ +async function updateMigrationState( + mongo: Db, + id: number, + state: MIGRATION_STATE.FINISHED | MIGRATION_STATE.FAILED, + now: Date +) { + const result = await collection(mongo).findOneAndUpdate( + { id }, + { + $set: { + state, + updatedAt: now, + }, + $unset: { + clientID: "", + }, + }, + { + // False to return the updated document instead of the original document. + returnOriginal: false, + } + ); + + return result.value || null; +} + +export async function finishMigration(mongo: Db, id: number, now = new Date()) { + return updateMigrationState(mongo, id, MIGRATION_STATE.FINISHED, now); +} + +export async function failMigration(mongo: Db, id: number, now = new Date()) { + return updateMigrationState(mongo, id, MIGRATION_STATE.FAILED, now); +} + +export async function retrieveAllMigrationRecords(mongo: Db) { + const cursor = await collection(mongo) + .find({}) + .sort({ id: 1 }); + return cursor.toArray(); +} diff --git a/src/core/server/models/queries/queries.ts b/src/core/server/models/queries/queries.ts index d8f80ead3..45c727b01 100644 --- a/src/core/server/models/queries/queries.ts +++ b/src/core/server/models/queries/queries.ts @@ -3,12 +3,7 @@ import { Db, MongoError } from "mongodb"; import { waitFor } from "coral-common/helpers"; import logger from "coral-server/logger"; -import { - createCollection, - createIndexFactory, -} from "coral-server/models/helpers"; - -const collection = createCollection>("queries"); +import { queries as collection } from "coral-server/services/mongodb/collections"; export interface PersistedQuery { id: string; @@ -19,13 +14,6 @@ export interface PersistedQuery { version: string; } -export async function createQueriesIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ id: 1 }, { unique: true }); -} - export async function primeQueries( mongo: Db, queries: PersistedQuery[], diff --git a/src/core/server/models/story/counts/index.ts b/src/core/server/models/story/counts/index.ts index 5415b33c2..11714291f 100644 --- a/src/core/server/models/story/counts/index.ts +++ b/src/core/server/models/story/counts/index.ts @@ -13,28 +13,12 @@ import { CommentStatusCounts, createEmptyCommentStatusCounts, } from "coral-server/models/comment/helpers"; -import { - createCollection, - createIndexFactory, -} from "coral-server/models/helpers"; import { retrieveStory, Story } from "coral-server/models/story"; +import { stories as collection } from "coral-server/services/mongodb/collections"; import { AugmentedRedis } from "coral-server/services/redis"; import { updateSharedCommentCounts } from "./shared"; -/** - * collection provides a reference to the stories collection used by the - * counting system. - */ -const collection = createCollection("stories"); - -export async function createStoryCountIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // { createdAt } - await createIndex({ tenantID: 1, createdAt: 1 }, { background: true }); -} - /** * CommentModerationCountsPerQueue stores the number of Comments that exist in * each of the Moderation Queues. diff --git a/src/core/server/models/story/counts/shared.ts b/src/core/server/models/story/counts/shared.ts index fe369bf99..5f2e956da 100644 --- a/src/core/server/models/story/counts/shared.ts +++ b/src/core/server/models/story/counts/shared.ts @@ -8,12 +8,11 @@ import { CommentStatusCounts, createEmptyCommentStatusCounts, } from "coral-server/models/comment/helpers"; -import { createCollection } from "coral-server/models/helpers"; -import { Story } from "coral-server/models/story"; import { CommentModerationCountsPerQueue, StoryCounts, } from "coral-server/models/story/counts"; +import { stories as collection } from "coral-server/services/mongodb/collections"; import { AugmentedPipeline, AugmentedRedis } from "coral-server/services/redis"; import { @@ -46,12 +45,6 @@ const commentCountsModerationQueueTotalKey = (tenantID: string) => const commentCountsModerationQueueQueuesKey = (tenantID: string) => `${tenantID}:commentCounts:moderationQueue:queues`; -/** - * collection provides a reference to the stories collection used by the - * counting system. - */ -const collection = createCollection("stories"); - /** * recalculateSharedModerationQueueQueueCounts will reset the counts stored for * this Tenant. diff --git a/src/core/server/models/story/index.ts b/src/core/server/models/story/index.ts index ce45cdd21..77b440018 100644 --- a/src/core/server/models/story/index.ts +++ b/src/core/server/models/story/index.ts @@ -14,16 +14,14 @@ import { import { Connection, ConnectionInput, - createConnectionOrderVariants, - createIndexFactory, Query, resolveConnection, } from "coral-server/models/helpers"; import { GlobalModerationSettings } from "coral-server/models/settings"; import { TenantResource } from "coral-server/models/tenant"; +import { stories as collection } from "coral-server/services/mongodb/collections"; import { createEmptyCommentStatusCounts } from "../comment/helpers"; -import { createCollection } from "../helpers/collection"; import { createEmptyCommentModerationQueueCounts, StoryCommentCounts, @@ -32,8 +30,6 @@ import { export * from "./counts"; export * from "./helpers"; -const collection = createCollection("stories"); - export type StorySettings = DeepPartial< Pick & GlobalModerationSettings >; @@ -81,40 +77,6 @@ export interface Story extends TenantResource { createdAt: Date; } -export async function createStoryIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE { url } - await createIndex({ tenantID: 1, url: 1 }, { unique: true }); - - // TEXT { $**, createdAt } - await createIndex( - { tenantID: 1, "$**": "text", createdAt: -1 }, - { background: true } - ); - - const variants = createConnectionOrderVariants>( - [{ createdAt: -1 }], - { background: true } - ); - - // Story Connection pagination. - // { ...connectionParams } - await variants(createIndex, { - tenantID: 1, - }); - - // Closed At ordered Story Connection pagination. - // { closedAt, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - closedAt: 1, - }); -} - export interface UpsertStoryInput { id?: string; url: string; diff --git a/src/core/server/models/tenant/tenant.ts b/src/core/server/models/tenant/tenant.ts index 5216ca047..f1f1288c2 100644 --- a/src/core/server/models/tenant/tenant.ts +++ b/src/core/server/models/tenant/tenant.ts @@ -9,15 +9,11 @@ import { GQLMODERATION_MODE, GQLSettings, } from "coral-server/graph/tenant/schema/__generated__/types"; -import { - createCollection, - createIndexFactory, -} from "coral-server/models/helpers"; import { Settings } from "coral-server/models/settings"; import { I18n } from "coral-server/services/i18n"; -import { generateSSOKey, getDefaultReactionConfiguration } from "./helpers"; +import { tenants as collection } from "coral-server/services/mongodb/collections"; -const collection = createCollection("tenants"); +import { generateSSOKey, getDefaultReactionConfiguration } from "./helpers"; /** * TenantResource references a given resource that should be owned by a specific @@ -46,16 +42,6 @@ export interface TenantSettings */ export type Tenant = Settings & TenantSettings; -export async function createTenantIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ id: 1 }, { unique: true }); - - // UNIQUE { domain } - await createIndex({ domain: 1 }, { unique: true }); -} - /** * CreateTenantInput is the set of properties that can be set when a given * Tenant is created. The remainder of the properties are set from defaults and diff --git a/src/core/server/models/user/user.ts b/src/core/server/models/user/user.ts index ea25400e4..a3e039189 100644 --- a/src/core/server/models/user/user.ts +++ b/src/core/server/models/user/user.ts @@ -33,19 +33,15 @@ import logger from "coral-server/logger"; import { Connection, ConnectionInput, - createCollection, - createConnectionOrderVariants, - createIndexFactory, Query, resolveConnection, } from "coral-server/models/helpers"; import { TenantResource } from "coral-server/models/tenant"; import { DigestibleTemplate } from "coral-server/queue/tasks/mailer/templates"; +import { users as collection } from "coral-server/services/mongodb/collections"; import { getLocalProfile, hasLocalProfile } from "./helpers"; -const collection = createCollection("users"); - export interface LocalProfile { type: "local"; id: string; @@ -297,10 +293,8 @@ export interface UserStatus { /** * premod stores whether a user is set to mandatory premod and history of * premod status. - * - * FIXME: (wyattjoh) set defaults during migration */ - premod?: PremodStatus; + premod: PremodStatus; } /** @@ -436,80 +430,6 @@ export interface User extends TenantResource { deletedAt?: Date; } -export async function createUserIndexes(mongo: Db) { - const createIndex = createIndexFactory(collection(mongo)); - - // UNIQUE { id } - await createIndex({ tenantID: 1, id: 1 }, { unique: true }); - - // UNIQUE - PARTIAL { email } - await createIndex( - { tenantID: 1, email: 1 }, - { unique: true, partialFilterExpression: { email: { $exists: true } } } - ); - - // UNIQUE { profiles.type, profiles.id } - await createIndex( - { tenantID: 1, "profiles.type": 1, "profiles.id": 1 }, - { - unique: true, - partialFilterExpression: { "profiles.id": { $exists: true } }, - } - ); - - // { profiles } - await createIndex( - { tenantID: 1, profiles: 1, email: 1 }, - { - partialFilterExpression: { profiles: { $exists: true } }, - background: true, - } - ); - - // TEXT { id, username, email, createdAt } - await createIndex( - { - tenantID: 1, - id: "text", - username: "text", - email: "text", - createdAt: -1, - }, - { background: true } - ); - - const variants = createConnectionOrderVariants>( - [{ createdAt: -1 }], - { background: true } - ); - - // User Connection pagination. - // { ...connectionParams } - await variants(createIndex, { - tenantID: 1, - }); - - // Role based User Connection pagination. - // { role, ...connectionParams } - await variants(createIndex, { - tenantID: 1, - role: 1, - }); - - // Suspension based User Connection pagination. - await variants(createIndex, { - tenantID: 1, - "status.suspension.history.from.start": 1, - "status.suspension.history.from.finish": 1, - }); - - // Ban based User Connection pagination. - await variants(createIndex, { - tenantID: 1, - "status.ban.active": 1, - }); -} - function hashPassword(password: string): Promise { return bcrypt.hash(password, 10); } @@ -1460,8 +1380,7 @@ export async function premodUser( // Check to see if the user is already banned. const premod = consolidateUserPremodStatus(user.status.premod); - // FIXME: (wyattjoh) once migration has been performed, remove check - if (premod && premod.active) { + if (premod.active) { throw new UserAlreadyPremoderated(); } @@ -1873,8 +1792,7 @@ export function consolidateUserSuspensionStatus( export interface ConsolidatedUserStatus { suspension: ConsolidatedSuspensionStatus; ban: ConsolidatedBanStatus; - // FIXME: (wyattjoh) once migration has been performed, make required - premod?: ConsolidatedPremodStatus; + premod: ConsolidatedPremodStatus; } export function consolidateUserStatus( diff --git a/src/core/server/services/comments/pipeline/phases/preModerateUser.ts b/src/core/server/services/comments/pipeline/phases/preModerateUser.ts index c375d5d86..c517befdb 100644 --- a/src/core/server/services/comments/pipeline/phases/preModerateUser.ts +++ b/src/core/server/services/comments/pipeline/phases/preModerateUser.ts @@ -8,8 +8,7 @@ import { export const premodUser: IntermediateModerationPhase = ({ author, }): IntermediatePhaseResult | void => { - // FIXME: (wyattjoh) once migration has been performed, remove check - if (author.status.premod && author.status.premod.active) { + if (author.status.premod.active) { return { status: GQLCOMMENT_STATUS.PREMOD, }; diff --git a/src/core/server/services/migrate/error.ts b/src/core/server/services/migrate/error.ts new file mode 100644 index 000000000..d207da4d5 --- /dev/null +++ b/src/core/server/services/migrate/error.ts @@ -0,0 +1,63 @@ +// tslint:disable: max-classes-per-file + +import VError from "verror"; + +import { MigrationRecord } from "coral-server/models/migration"; + +export class MigrationError extends VError { + public readonly tenantID: string; + public readonly reason: string; + public readonly affectedCollection?: string; + public readonly affectedIDs?: string[]; + + constructor( + tenantID: string, + reason: string, + affectedCollection?: string, + affectedIDs?: string[] + ) { + super( + { + name: "MigrationError", + info: { + tenantID, + affectedCollection, + affectedIDs, + }, + }, + 'MigrationError: "%s"', + reason + ); + + this.tenantID = tenantID; + this.reason = reason; + this.affectedCollection = affectedCollection; + this.affectedIDs = affectedIDs; + } +} + +export class FailedMigrationDetectedError extends VError { + constructor(record: MigrationRecord) { + super( + { + name: "FailedMigrationDetectedError", + info: record, + }, + 'FailedMigrationDetectedError: migration "%d" failed, remove this document to restart the migration process', + record.id + ); + } +} + +export class InProgressMigrationDetectedError extends VError { + constructor(record: MigrationRecord) { + super( + { + name: "InProgressMigrationDetectedError", + info: record, + }, + 'InProgressMigrationDetectedError: migration "%d" was in progress', + record.id + ); + } +} diff --git a/src/core/server/services/migrate/index.ts b/src/core/server/services/migrate/index.ts new file mode 100644 index 000000000..ef77b4127 --- /dev/null +++ b/src/core/server/services/migrate/index.ts @@ -0,0 +1,2 @@ +export { default as Migration } from "./migration"; +export { default as MigrationManager } from "./manager"; diff --git a/src/core/server/models/helpers/indexing.ts b/src/core/server/services/migrate/indexing.ts similarity index 84% rename from src/core/server/models/helpers/indexing.ts rename to src/core/server/services/migrate/indexing.ts index 77a4712f6..43d3b569b 100644 --- a/src/core/server/models/helpers/indexing.ts +++ b/src/core/server/services/migrate/indexing.ts @@ -1,5 +1,6 @@ import { merge } from "lodash"; import { Collection, IndexOptions } from "mongodb"; +import now from "performance-now"; import { Writable } from "coral-common/types"; import logger from "coral-server/logger"; @@ -32,8 +33,13 @@ export function createIndexFactory( ) => { try { // Try to create the index. + const start = now(); + log.debug({ indexSpec, indexOptions }, "creating index"); const indexName = await collection.createIndex(indexSpec, indexOptions); - log.debug({ indexName, indexSpec, indexOptions }, "index was created"); + log.debug( + { indexName, indexSpec, indexOptions, took: Math.round(now() - start) }, + "index was created" + ); // Match the interface from the `createIndex` function by returning the // index name. @@ -48,11 +54,11 @@ export function createIndexFactory( } export function createConnectionOrderVariants( + createIndex: IndexCreationFunction, variants: Array>, - indexOptions: IndexOptions = {} + indexOptions: IndexOptions = { background: true } ) { return async ( - createIndex: IndexCreationFunction, indexSpec: IndexSpecification, variantIndexOptions: IndexOptions = {} ) => { @@ -68,9 +74,6 @@ export function createConnectionOrderVariants( merge({}, indexOptions, variantIndexOptions) ); - // Create a raw index without the variants applied. - await createIndex(indexSpec, merge({}, indexOptions, variantIndexOptions)); - // Create all the variants. for (const variant of variants) { await createIndexVariant(variant); diff --git a/src/core/server/services/migrate/manager.ts b/src/core/server/services/migrate/manager.ts new file mode 100644 index 000000000..41acc779c --- /dev/null +++ b/src/core/server/services/migrate/manager.ts @@ -0,0 +1,238 @@ +import fs from "fs-extra"; +import { Db } from "mongodb"; +import path from "path"; +import now from "performance-now"; +import uuid from "uuid"; + +import logger from "coral-server/logger"; +import { + failMigration, + finishMigration, + MIGRATION_STATE, + retrieveAllMigrationRecords, + startMigration, +} from "coral-server/models/migration"; +import TenantCache from "coral-server/services/tenant/cache"; + +import { + FailedMigrationDetectedError, + InProgressMigrationDetectedError, +} from "./error"; +import Migration from "./migration"; + +// Extract the id from the filename with this regex. +const fileNamePattern = /^(\d+)_([\S_]+)\.[tj]s$/; + +export default class Manager { + private clientID: string; + private migrations: Migration[]; + private tenants: TenantCache; + private ran: boolean = false; + + constructor(tenants: TenantCache) { + this.clientID = uuid.v4(); + this.migrations = []; + this.tenants = tenants; + + const fileNames = fs.readdirSync(path.join(__dirname, "migrations")); + for (const fileName of fileNames) { + // Test to see if this fileName is one of our migrations. + if (!fileNamePattern.test(fileName)) { + if (fileName.endsWith(".map")) { + // This was a mapping file (in production), skip this. + continue; + } + + logger.warn( + { fileName }, + "found a file in the migrations folder that did not have an expected format" + ); + continue; + } + + // Load the migration. + const filePath = path.join(__dirname, "migrations", fileName); + const m = require(filePath); + + // Parse the timestamp out of the migration filename. + const matches = fileName.match(fileNamePattern); + if (!matches || matches.length !== 3) { + throw new Error("fileName format is invalid"); + } + const id = parseInt(matches[1], 10); + + // Create the migration instance. + const migration = new m.default(id, matches[2]); + + // Insert the migration into the migrations array. + if (!(migration instanceof Migration)) { + throw new Error(`migration at ${filePath} did not export a Migration`); + } + + this.migrations.push(migration); + } + + // Sort the migrations. + this.migrations.sort((a, b) => { + if (a.id < b.id) { + return -1; + } + + if (a.id > b.id) { + return 1; + } + + return 0; + }); + } + + /** + * pending will return the pending migrations that need to be completed. + * + * @param mongo the database handle to use to get the migrations + */ + private async pending(mongo: Db): Promise { + // Get all the migration records in the database. + const records = await retrieveAllMigrationRecords(mongo); + + // Check to see if any of the migrations have failed or are in progress. + for (const record of records) { + switch (record.state) { + case MIGRATION_STATE.FAILED: + throw new FailedMigrationDetectedError(record); + case MIGRATION_STATE.STARTED: + throw new InProgressMigrationDetectedError(record); + default: + break; + } + } + + return this.migrations.filter(migration => { + // Find the record based on the migration. + const record = records.find(({ id }) => migration.id === id); + if (record) { + // The record exists, so it isn't pending, it's already finished. + return false; + } + + // A record of the migration does not exist, so mark it as pending. + return true; + }); + } + + private async currentMigration(mongo: Db) { + const records = await retrieveAllMigrationRecords(mongo); + return records.length > 0 ? records[records.length - 1] : null; + } + + public async executePendingMigrations(mongo: Db) { + // Error out if this is ran twice. + if (this.ran) { + throw new Error("pending migrations have already been executed"); + } + + // Mark the migrations as ran. + this.ran = true; + + // Check the current migration id. + let currentMigration = await this.currentMigration(mongo); + + // Determine which migrations need to be ran. + const pending = await this.pending(mongo); + if (pending.length === 0) { + logger.info( + { + currentMigrationID: currentMigration ? currentMigration.id : null, + }, + "there was no pending migrations to run" + ); + return; + } + + logger.info({ pending: pending.length }, "executing pending migrations"); + + const migrationsStartTime = now(); + + for (const migration of pending) { + let log = logger.child( + { + migrationName: migration.name, + migrationID: migration.id, + }, + true + ); + + // Mark the migration as started. + const record = await startMigration(mongo, migration.id, this.clientID); + if (record.clientID !== this.clientID) { + throw new InProgressMigrationDetectedError(record); + } + + // Apply any index changes for the migration. + if (migration.indexes) { + const migrationStartTime = now(); + log.info("starting index migration"); + await migration.indexes(mongo); + const executionTime = Math.round(now() - migrationStartTime); + log.info({ executionTime }, "finished index migration"); + } + + if (migration.up) { + // The migration provides an up method, we should run this per Tenant. + for await (const tenant of this.tenants) { + log = log.child({ tenantID: tenant.id }, true); + + const migrationStartTime = now(); + log.info("starting migration"); + + try { + // Up the migration. + await migration.up(mongo, tenant.id); + + // Test the migration. + if (migration.test) { + await migration.test(mongo, tenant.id); + } + } catch (err) { + // The migration or test has failed, try to roll back the operation. + if (migration.down) { + log.error({ err }, "migration has failed, attempting rollback"); + + // Attempt the down migration. + await migration.down(mongo, tenant.id); + } else { + log.error( + { err }, + "migration has failed, and does not have a down method available, migration will not be rolled back" + ); + } + + // Mark the migration as failed. + await failMigration(mongo, migration.id); + + // Rethrow the error here to cause the application to crash. + throw err; + } + + const executionTime = Math.round(now() - migrationStartTime); + log.info({ executionTime }, "finished migration"); + } + } + + // Mark the migration as completed. + await finishMigration(mongo, migration.id); + } + + const finishTime = Math.round(now() - migrationsStartTime); + + currentMigration = await this.currentMigration(mongo); + + logger.info( + { + finishTime, + currentMigrationID: currentMigration ? currentMigration.id : null, + }, + "finished running pending migrations" + ); + } +} diff --git a/src/core/server/services/migrate/migration.ts b/src/core/server/services/migrate/migration.ts new file mode 100644 index 000000000..3d2505f61 --- /dev/null +++ b/src/core/server/services/migrate/migration.ts @@ -0,0 +1,35 @@ +import Logger from "bunyan"; +import { Db } from "mongodb"; + +import logger from "coral-server/logger"; + +interface Migration { + indexes?(mongo: Db): Promise; + up?(mongo: Db, tenantID: string): Promise; + test?(mongo: Db, tenantID: string): Promise; + down?(mongo: Db, tenantID: string): Promise; +} + +abstract class Migration { + public readonly name: string; + public readonly id: number; + public readonly logger: Logger; + + constructor(version: number, name: string) { + this.id = version; + this.name = name; + this.logger = logger.child( + { + migrationName: this.name, + migrationVersion: this.id, + }, + true + ); + } + + protected log(tenantID: string) { + return this.logger.child({ tenantID }, true); + } +} + +export default Migration; diff --git a/src/core/server/services/migrate/migration_sample.ts b/src/core/server/services/migrate/migration_sample.ts new file mode 100644 index 000000000..0c96026b4 --- /dev/null +++ b/src/core/server/services/migrate/migration_sample.ts @@ -0,0 +1,13 @@ +import { Db } from "mongodb"; + +// Use the following collections reference to interact with specific +// collections. +// import collections from "coral-server/services/mongodb/collections"; + +import Migration from "coral-server/services/migrate/migration"; + +export default class extends Migration { + public async up(mongo: Db, tenantID: string) { + throw new Error("migration not implemented"); + } +} diff --git a/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts b/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts new file mode 100644 index 000000000..82bdfcc1a --- /dev/null +++ b/src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts @@ -0,0 +1,77 @@ +import { Db } from "mongodb"; + +import collections from "coral-server/services/mongodb/collections"; + +import { MigrationError } from "../error"; +import Migration from "../migration"; + +export default class extends Migration { + public async down(mongo: Db, tenantID: string) { + // Remove the premod user status from all users on this Tenant. + const result = await collections.users(mongo).updateMany( + { + tenantID, + "status.premod": { $ne: null }, + }, + { + $unset: { + "status.premod": "", + }, + } + ); + + this.log(tenantID).warn( + { matchedCount: result.matchedCount, modifiedCount: result.matchedCount }, + "cleared the premod status from users" + ); + } + + public async test(mongo: Db, tenantID: string) { + // Find all the users that still have premod status unset. + const cursor = await collections + .users(mongo) + .find({ + "status.premod": null, + tenantID, + }) + .project({ id: 1 }); + + // Count them. + const users = await cursor.toArray(); + const count = users.length; + if (count === 0) { + this.log(tenantID).info("all users migrated successfully"); + return; + } + + throw new MigrationError( + tenantID, + "found users that were not updated", + "users", + users.map(({ id }) => id) + ); + } + + public async up(mongo: Db, tenantID: string) { + // Migrate users to include the premod status. + const result = await collections.users(mongo).updateMany( + { + "status.premod": null, + tenantID, + }, + { + $set: { + "status.premod": { + active: false, + history: [], + }, + }, + } + ); + + this.log(tenantID).info( + { matchedCount: result.matchedCount, modifiedCount: result.matchedCount }, + "updated user premod status" + ); + } +} diff --git a/src/core/server/services/migrate/migrations/1569612830133_indexes.ts b/src/core/server/services/migrate/migrations/1569612830133_indexes.ts new file mode 100644 index 000000000..82c458ecc --- /dev/null +++ b/src/core/server/services/migrate/migrations/1569612830133_indexes.ts @@ -0,0 +1,342 @@ +import { Db } from "mongodb"; + +import Migration from "coral-server/services/migrate/migration"; +import collections from "coral-server/services/mongodb/collections"; + +import { createConnectionOrderVariants, createIndexFactory } from "../indexing"; + +async function createMigrationRecordIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.migrations(mongo)); + + // UNIQUE { id } + await createIndex({ id: 1 }, { unique: true }); +} + +async function createUserIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.users(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // UNIQUE - PARTIAL { email } + await createIndex( + { tenantID: 1, email: 1 }, + { unique: true, partialFilterExpression: { email: { $exists: true } } } + ); + + // UNIQUE { profiles.type, profiles.id } + await createIndex( + { tenantID: 1, "profiles.type": 1, "profiles.id": 1 }, + { + unique: true, + partialFilterExpression: { "profiles.id": { $exists: true } }, + } + ); + + // { profiles } + await createIndex( + { tenantID: 1, profiles: 1, email: 1 }, + { + partialFilterExpression: { profiles: { $exists: true } }, + background: true, + } + ); + + // TEXT { id, username, email, createdAt } + await createIndex( + { + tenantID: 1, + id: "text", + username: "text", + email: "text", + createdAt: -1, + }, + { background: true } + ); + + const variants = createConnectionOrderVariants( + createIndex, + [{ createdAt: -1 }], + { background: true } + ); + + // User Connection pagination. + // { ...connectionParams } + await variants({ + tenantID: 1, + }); + + // Role based User Connection pagination. + // { role, ...connectionParams } + await variants({ + tenantID: 1, + role: 1, + }); + + // Suspension based User Connection pagination. + await variants({ + tenantID: 1, + "status.suspension.history.from.start": 1, + "status.suspension.history.from.finish": 1, + }); + + // Ban based User Connection pagination. + await variants({ + tenantID: 1, + "status.ban.active": 1, + }); +} + +async function createInviteIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.invites(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // UNIQUE { email } + await createIndex({ tenantID: 1, email: 1 }, { unique: true }); +} + +async function createTenantIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.tenants(mongo)); + + // UNIQUE { id } + await createIndex({ id: 1 }, { unique: true }); + + // UNIQUE { domain } + await createIndex({ domain: 1 }, { unique: true }); +} + +async function createStoryIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.stories(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // UNIQUE { url } + await createIndex({ tenantID: 1, url: 1 }, { unique: true }); + + // TEXT { $**, createdAt } + await createIndex( + { tenantID: 1, "$**": "text", createdAt: -1 }, + { background: true } + ); + + const variants = createConnectionOrderVariants( + createIndex, + [{ createdAt: -1 }], + { background: true } + ); + + // Story Connection pagination. + // { ...connectionParams } + await variants({ + tenantID: 1, + }); + + // Closed At ordered Story Connection pagination. + // { closedAt, ...connectionParams } + await variants({ + tenantID: 1, + closedAt: 1, + }); +} + +async function createStoryCountIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.stories(mongo)); + + // { createdAt } + await createIndex({ tenantID: 1, createdAt: 1 }, { background: true }); +} + +async function createQueriesIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.queries(mongo)); + + // UNIQUE { id } + await createIndex({ id: 1 }, { unique: true }); +} + +async function createCommentActionIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.commentActions(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // { actionType, commentID, userID } + await createIndex( + { tenantID: 1, actionType: 1, commentID: 1, userID: 1 }, + { background: true } + ); + + const variants = createConnectionOrderVariants( + createIndex, + [{ createdAt: -1 }], + { background: true } + ); + + // Connection pagination. + // { ...connectionParams } + await variants({ + tenantID: 1, + actionType: 1, + commentID: 1, + }); +} + +async function createCommentModerationActionIndexes(mongo: Db) { + const createIndex = createIndexFactory( + collections.commentModerationActions(mongo) + ); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + const createVariants = createConnectionOrderVariants(createIndex, [ + { createdAt: -1 }, + ]); + + // { moderatorID, ...connectionParams } + await createVariants({ + moderatorID: 1, + }); +} + +async function createCommentIndexes(mongo: Db) { + const createIndex = createIndexFactory(collections.comments(mongo)); + + // UNIQUE { id } + await createIndex({ tenantID: 1, id: 1 }, { unique: true }); + + // Facility for counting the tags against a story. + await createIndex( + { + tenantID: 1, + storyID: 1, + "tags.type": 1, + status: 1, + }, + { + background: true, + partialFilterExpression: { + "tags.type": { $exists: true }, + }, + } + ); + + const streamVariants = createConnectionOrderVariants(createIndex, [ + { createdAt: -1 }, + { createdAt: 1 }, + { childCount: -1, createdAt: -1 }, + { "actionCounts.REACTION": -1, createdAt: -1 }, + ]); + + // Story based Comment Connection pagination. + // { storyID, ...connectionParams } + await streamVariants({ + tenantID: 1, + storyID: 1, + status: 1, + }); + + // Story + Reply based Comment Connection pagination. + // { storyID, ...connectionParams } + await streamVariants({ + tenantID: 1, + storyID: 1, + parentID: 1, + status: 1, + }); + + // Author based Comment Connection pagination. + // { authorID, ...connectionParams } + await streamVariants({ + tenantID: 1, + authorID: 1, + status: 1, + }); + + // Tag based Comment Connection pagination. + // { tags.type, ...connectionParams } + await streamVariants({ + tenantID: 1, + storyID: 1, + "tags.type": 1, + }); + + const adminVariants = createConnectionOrderVariants( + createIndex, + [{ createdAt: -1 }, { createdAt: 1 }], + { background: true } + ); + + // Moderation based Comment Connection pagination. + // { storyID, ...connectionParams } + await adminVariants({ + tenantID: 1, + status: 1, + }); + + // Story based Comment Connection pagination that are flagged. + // { storyID, ...connectionParams } + await adminVariants({ + tenantID: 1, + storyID: 1, + status: 1, + "actionCounts.FLAG": 1, + }); + + // Author based Comment Connection pagination. + // { authorID, ...connectionParams } + await adminVariants({ + tenantID: 1, + authorID: 1, + }); +} + +type IndexCreationFunction = (mongo: Db) => Promise; + +const indexes: Array<[string, IndexCreationFunction]> = [ + ["migrations", createMigrationRecordIndexes], + ["users", createUserIndexes], + ["invites", createInviteIndexes], + ["tenants", createTenantIndexes], + ["comments", createCommentIndexes], + ["stories", createStoryIndexes], + ["stories", createStoryCountIndexes], + ["commentActions", createCommentActionIndexes], + ["commentModerationActions", createCommentModerationActionIndexes], + ["queries", createQueriesIndexes], +]; + +export default class extends Migration { + /** + * ensureIndexes will ensure that all indexes have been created. + * + * @param mongo a MongoDB Database Connection + */ + private async ensureIndexes(mongo: Db) { + this.logger.info( + { indexGroupCount: indexes.length }, + "now ensuring indexes are created" + ); + + // For each of the index functions, call it. + for (const [indexGroup, indexFunction] of indexes) { + this.logger.info({ indexGroup }, "ensuring indexes are created"); + await indexFunction(mongo); + this.logger.info({ indexGroup }, "indexes have been created"); + } + + this.logger.info("all indexes have been created"); + } + + public async indexes(mongo: Db) { + // Drop existing indexes on managed collections so we can re-create them. + for (const collection of Object.values(collections)) { + await collection(mongo).dropIndexes(); + } + + // Re-create the indexes for each collection now. + await this.ensureIndexes(mongo); + } +} diff --git a/src/core/server/services/mongodb/collections.ts b/src/core/server/services/mongodb/collections.ts new file mode 100644 index 000000000..b07996eed --- /dev/null +++ b/src/core/server/services/mongodb/collections.ts @@ -0,0 +1,45 @@ +import { createCollection } from "coral-server/models/helpers"; + +import { CommentAction } from "coral-server/models/action/comment"; +import { CommentModerationAction } from "coral-server/models/action/moderation/comment"; +import { Comment } from "coral-server/models/comment"; +import { Invite } from "coral-server/models/invite"; +import { MigrationRecord } from "coral-server/models/migration"; +import { PersistedQuery } from "coral-server/models/queries"; +import { Story } from "coral-server/models/story"; +import { Tenant } from "coral-server/models/tenant"; +import { User } from "coral-server/models/user"; + +export const users = createCollection("users"); + +export const invites = createCollection("invites"); + +export const tenants = createCollection("tenants"); + +export const comments = createCollection("comments"); + +export const stories = createCollection("stories"); + +export const commentActions = createCollection("commentActions"); + +export const commentModerationActions = createCollection< + CommentModerationAction +>("commentModerationActions"); + +export const queries = createCollection("queries"); + +export const migrations = createCollection("migrations"); + +const collections = { + users, + invites, + tenants, + comments, + stories, + commentActions, + commentModerationActions, + queries, + migrations, +}; + +export default collections; diff --git a/src/core/server/services/mongodb/indexes.ts b/src/core/server/services/mongodb/indexes.ts deleted file mode 100644 index 98fe3f99b..000000000 --- a/src/core/server/services/mongodb/indexes.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Db } from "mongodb"; - -import logger from "coral-server/logger"; -import { createCommentActionIndexes } from "coral-server/models/action/comment"; -import { createCommentModerationActionIndexes } from "coral-server/models/action/moderation/comment"; -import { createCommentIndexes } from "coral-server/models/comment"; -import { createInviteIndexes } from "coral-server/models/invite"; -import { createQueriesIndexes } from "coral-server/models/queries"; -import { - createStoryCountIndexes, - createStoryIndexes, -} from "coral-server/models/story"; -import { createTenantIndexes } from "coral-server/models/tenant"; -import { createUserIndexes } from "coral-server/models/user"; - -type IndexCreationFunction = (mongo: Db) => Promise; - -const indexes: Array<[string, IndexCreationFunction]> = [ - ["users", createUserIndexes], - ["invites", createInviteIndexes], - ["tenants", createTenantIndexes], - ["comments", createCommentIndexes], - ["stories", createStoryIndexes], - ["stories", createStoryCountIndexes], - ["commentActions", createCommentActionIndexes], - ["commentModerationActions", createCommentModerationActionIndexes], - ["queries", createQueriesIndexes], -]; - -/** - * ensureIndexes will ensure that all indexes have been created. - * - * @param mongo a MongoDB Database Connection - */ -export async function ensureIndexes(mongo: Db) { - logger.debug( - { indexGroupCount: indexes.length }, - "now ensuring indexes are created" - ); - - // For each of the index functions, call it. - for (const [indexGroup, indexFunction] of indexes) { - logger.debug({ indexGroup }, "ensuring indexes are created"); - await indexFunction(mongo); - logger.debug({ indexGroup }, "indexes have been created"); - } - - logger.debug("all indexes have been created"); -} diff --git a/src/core/server/services/tenant/cache/index.ts b/src/core/server/services/tenant/cache/index.ts index 7cba1229a..6b9ccc64d 100644 --- a/src/core/server/services/tenant/cache/index.ts +++ b/src/core/server/services/tenant/cache/index.ts @@ -198,6 +198,8 @@ export default class TenantCache { yield tenant; } + + return; } // Caching must be disabled, so just grab all the tenants for this node and diff --git a/src/core/server/services/tenant/index.ts b/src/core/server/services/tenant/index.ts index eea30f6b1..b1f229565 100644 --- a/src/core/server/services/tenant/index.ts +++ b/src/core/server/services/tenant/index.ts @@ -87,8 +87,6 @@ export async function install( throw new TenantInstalledAlreadyError(); } - // TODO: (wyattjoh) perform any pending migrations. - logger.info("installing tenant"); // Create the Tenant. diff --git a/src/core/server/services/users/delete.ts b/src/core/server/services/users/delete.ts index e44023d4d..b53da135f 100644 --- a/src/core/server/services/users/delete.ts +++ b/src/core/server/services/users/delete.ts @@ -1,22 +1,10 @@ import { Collection, Db } from "mongodb"; -import { CommentAction } from "coral-server/models/action/comment"; -import { createCollection } from "coral-server/models/helpers"; import { Story } from "coral-server/models/story"; -import { Tenant } from "coral-server/models/tenant"; -import { User } from "coral-server/models/user"; +import collections from "coral-server/services/mongodb/collections"; const BATCH_SIZE = 500; -// TODO: extract this out to a separate file so it can be re-used elsewhere -const collections = { - users: createCollection("users"), - comments: createCollection("comments"), - stories: createCollection("stories"), - tenants: createCollection("tenants"), - commentActions: createCollection("commentActions"), -}; - async function executeBulkOperations( collection: Collection, operations: any[] diff --git a/src/core/server/services/users/index.ts b/src/core/server/services/users/index.ts index 7e6497cd2..a358a3b5a 100644 --- a/src/core/server/services/users/index.ts +++ b/src/core/server/services/users/index.ts @@ -820,8 +820,7 @@ export async function premod( // Check to see if the User is currently banned. const premodStatus = consolidateUserPremodStatus(targetUser.status.premod); - // FIXME: (wyattjoh) once migration has been performed, remove check - if (premodStatus && premodStatus.active) { + if (premodStatus.active) { throw new UserAlreadyPremoderated(); } @@ -845,8 +844,7 @@ export async function removePremod( // Check to see if the User is currently suspended. const premodStatus = consolidateUserPremodStatus(targetUser.status.premod); - // FIXME: (wyattjoh) once migration has been performed, remove check - if (!premodStatus || !premodStatus.active) { + if (!premodStatus.active) { // The user is not premodded currently, just return the user because we // don't have to do anything. return targetUser;