[CORL-649] Migrations (#2597)

* feat: added migration framework

* chore: added premod user status migration

* feat: enhanced error handling of migrations

* fix: added missing argument from abstract method

* fix: another templating blunder

* fix: removed debug code

* feat: enhanced migration tracking

* fix: remove skipping migrations

* feat: moved indexing to migration system

* fix: linting

README.md

@@ -128,7 +128,7 @@ If you don't already have these databases running, you can execute the following
 assuming you have Docker installed on your local machine:
 
 ```bash
-docker run -d -p 27017:27017 --restart always --name mongo mongo:3.6
+docker run -d -p 27017:27017 --restart always --name mongo mongo:4.2
 docker run -d -p 6379:6379 --restart always --name redis redis:3.2
 ```
 
@@ -624,12 +624,6 @@ the variables in a `.env` file in the root of the project in a simple
 - `DISABLE_TENANT_CACHING` - When `true`, all tenants will be loaded from the
   database when needed rather than keeping a in-memory copy in sync via
   published events on Redis. (Default `false`)
-- `DISABLE_MONGODB_AUTOINDEXING` - When `true`, Coral will not perform indexing
-  operations when it starts up. This can be desired when you've already
-  installed Coral on the target MongoDB, but want to improve start performance.
-  **You should not use this parameter unless you know what you're doing! Upgrades
-  may introduce additional indexes that the application relies on.**
-  (Default `false`)
 - `LOCALE` - Specify the default locale to use for all requests without a locale
   specified. (Default `en-US`)
 - `ENABLE_GRAPHIQL` - When `true`, it will enable the GraphiQL interface at `/graphiql`. **(🚨 Note 🚨) we do not recommend using this in production environments as it disables many safety features used by the application**. (Default `false`)

package.json

@@ -24,6 +24,7 @@
     "build:development": "NODE_ENV=development npm-run-all generate --parallel lint:client build:client build:server",
     "build:client": "ts-node --transpile-only ./scripts/build.ts",
     "build:server": "gulp server",
+    "migration:create": "ts-node --transpile-only ./scripts/migration/create.ts",
     "doctoc": "doctoc --title='## Table of Contents' --github README.md",
     "generate": "npm-run-all generate:css-types generate:schema generate:relay",
     "generate-persist": "npm-run-all generate:css-types generate:schema generate:relay-persist",

scripts/migration/create.ts

@@ -0,0 +1,52 @@
+#!/usr/bin/env ts-node
+
+/**
+ * This script can be invoked via:
+ *
+ *   npm run migration:create <migration name>
+ *
+ * To create new database migrations.
+ */
+
+// tslint:disable: no-console
+
+import fs from "fs-extra";
+import lodash from "lodash";
+import path from "path";
+
+const templateFilePath = path.resolve(
+  path.join(
+    __dirname,
+    "../../src/core/server/services/migrate/migration_sample.ts"
+  )
+);
+
+const argv = process.argv.slice(2);
+if (argv.length !== 1) {
+  console.error("usage: npm run migration:create <migration name>");
+  process.exit(1);
+}
+
+// Get the name of the new migration.
+const name = lodash.snakeCase(argv[0]);
+
+// Get the version of the new migration.
+const version = Date.now();
+
+// Get the filePath of the new migration.
+const filePath = path.resolve(
+  path.join(
+    __dirname,
+    `../../src/core/server/services/migrate/migrations/${version}_${name}.ts`
+  )
+);
+
+if (fs.existsSync(filePath)) {
+  console.error(`migration already exists at: ${filePath}`);
+  process.exit(1);
+}
+
+// Write the template out to the file.
+fs.copyFileSync(templateFilePath, filePath);
+
+console.log(`created new migration at: ${filePath}`);

src/core/client/admin/components/UserHistoryDrawer/UserDrawerAccountHistory.tsx

@@ -107,20 +107,17 @@ const UserDrawerAccountHistory: FunctionComponent<Props> = ({ user }) => {
       });
     });
 
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (user.status.premod) {
-      // Merge in all the premod history items.
-      user.status.premod.history.forEach(record => {
-        history.push({
-          kind: "premod",
-          action: {
-            action: record.active ? "created" : "removed",
-          },
-          date: new Date(record.createdAt),
-          takenBy: record.createdBy ? record.createdBy.username : system,
-        });
+    // Merge in all the premod history items.
+    user.status.premod.history.forEach(record => {
+      history.push({
+        kind: "premod",
+        action: {
+          action: record.active ? "created" : "removed",
+        },
+        date: new Date(record.createdAt),
+        takenBy: record.createdBy ? record.createdBy.username : system,
       });
-    }
+    });
 
     user.status.username.history.forEach((record, i) => {
       history.push({

src/core/client/admin/components/UserStatus/UserStatusChangeContainer.tsx

@@ -66,8 +66,7 @@ const UserStatusChangeContainer: FunctionComponent<Props> = props => {
   }, [user, removeUserSuspension]);
 
   const handlePremod = useCallback(() => {
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (user.status.premod && user.status.premod.active) {
+    if (user.status.premod.active) {
       return;
     }
     setShowPremod(true);
@@ -83,8 +82,7 @@ const UserStatusChangeContainer: FunctionComponent<Props> = props => {
   }, [setShowPremod]);
 
   const handleRemovePremod = useCallback(() => {
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (!user.status.premod || !user.status.premod.active) {
+    if (!user.status.premod.active) {
       return;
     }
     removeUserPremod({ userID: user.id });
@@ -138,8 +136,7 @@ const UserStatusChangeContainer: FunctionComponent<Props> = props => {
         onRemovePremod={handleRemovePremod}
         banned={user.status.ban.active}
         suspended={user.status.suspension.active}
-        // FIXME: (wyattjoh) once migration has been performed, remove check
-        premod={Boolean(user.status.premod && user.status.premod.active)}
+        premod={user.status.premod.active}
         fullWidth={fullWidth}
       >
         <UserStatusContainer user={user} />

src/core/server/app/handlers/api/install.ts

@@ -54,7 +54,7 @@ const TenantInstallBodySchema = Joi.object().keys({
 
 export type TenantInstallHandlerOptions = Pick<
   AppOptions,
-  "redis" | "mongo" | "config" | "mailerQueue" | "i18n"
+  "redis" | "mongo" | "config" | "mailerQueue" | "i18n" | "migrationManager"
 >;
 
 export const installHandler = ({
@@ -62,6 +62,7 @@ export const installHandler = ({
   redis,
   config,
   i18n,
+  migrationManager,
 }: TenantInstallHandlerOptions): RequestHandler => async (req, res, next) => {
   try {
     if (!req.coral) {
@@ -108,6 +109,9 @@ export const installHandler = ({
       req.coral.now
     );
 
+    // Execute pending migrations to get everything installed.
+    await migrationManager.executePendingMigrations(mongo);
+
     // Pull the user details out of the input for the user.
     const { email, username, password } = userInput;
 

src/core/server/app/index.ts

@@ -19,6 +19,7 @@ import { ScraperQueue } from "coral-server/queue/tasks/scraper";
 import { I18n } from "coral-server/services/i18n";
 import { JWTSigningConfig } from "coral-server/services/jwt";
 import { Metrics } from "coral-server/services/metrics";
+import { MigrationManager } from "coral-server/services/migrate";
 import { PersistedQueryCache } from "coral-server/services/queries";
 import { AugmentedRedis } from "coral-server/services/redis";
 import TenantCache from "coral-server/services/tenant/cache";
@@ -46,6 +47,7 @@ export interface AppOptions {
   scraperQueue: ScraperQueue;
   signingConfig: JWTSigningConfig;
   tenantCache: TenantCache;
+  migrationManager: MigrationManager;
 }
 
 /**

src/core/server/config.ts

@@ -204,13 +204,6 @@ const config = convict({
     env: "DISABLE_LIVE_UPDATES",
     arg: "disableLiveUpdates",
   },
-  disable_mongodb_autoindexing: {
-    doc: "Disables the creation of new MongoDB indexes",
-    format: Boolean,
-    default: false,
-    env: "DISABLE_MONGODB_AUTOINDEXING",
-    arg: "disableMongoDBAutoindexing",
-  },
   disable_client_routes: {
     doc:
       "Disables mounting of client routes for developing with Webpack Dev Server",

src/core/server/graph/tenant/resolvers/UserStatus.ts

@@ -31,8 +31,7 @@ export const UserStatus: Required<
     }
 
     // If they are set to mandatory premod, then mark it.
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (consolidatedStatus.premod && consolidatedStatus.premod.active) {
+    if (consolidatedStatus.premod.active) {
       statuses.push(GQLUSER_STATUS.PREMOD);
     }
 
@@ -55,17 +54,8 @@ export const UserStatus: Required<
     ...user.consolidateUserSuspensionStatus(suspension),
     userID,
   }),
-  // FIXME: (wyattjoh) once migration has been performed, return PremodStatusInput only
-  premod: ({ premod, userID }): PremodStatusInput | null => {
-    const status = user.consolidateUserPremodStatus(premod);
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (!status) {
-      return null;
-    }
-
-    return {
-      ...status,
-      userID,
-    };
-  },
+  premod: ({ premod, userID }): PremodStatusInput => ({
+    ...user.consolidateUserPremodStatus(premod),
+    userID,
+  }),
 };

src/core/server/graph/tenant/schema/schema.graphql

@@ -1541,10 +1541,8 @@ type UserStatus {
 
   """
   premod stores the user premod status as well as the history of changes.
-
-  FIXME: (wyattjoh) once migration has been performed, make non-nullable
   """
-  premod: PremodStatus @auth(roles: [ADMIN, MODERATOR])
+  premod: PremodStatus! @auth(roles: [ADMIN, MODERATOR])
 }
 
 """

src/core/server/index.ts

@@ -27,8 +27,8 @@ import {
   JWTSigningConfig,
 } from "coral-server/services/jwt";
 import { createMetrics } from "coral-server/services/metrics";
+import { MigrationManager } from "coral-server/services/migrate";
 import { createMongoDB } from "coral-server/services/mongodb";
-import { ensureIndexes } from "coral-server/services/mongodb/indexes";
 import { PersistedQueryCache } from "coral-server/services/queries";
 import {
   AugmentedRedis,
@@ -36,6 +36,7 @@ import {
   createRedisClient,
 } from "coral-server/services/redis";
 import TenantCache from "coral-server/services/tenant/cache";
+import { isInstalled } from "./services/tenant";
 
 export interface ServerOptions {
   /**
@@ -103,6 +104,9 @@ class Server {
   // server to handle persisted queries.
   private persistedQueryCache: PersistedQueryCache;
 
+  // migrationManager is the manager for performing migrations on Coral.
+  private migrationManager: MigrationManager;
+
   constructor(options: ServerOptions) {
     this.parentApp = express();
 
@@ -153,6 +157,9 @@ class Server {
       config
     );
 
+    // Create the migration manager.
+    this.migrationManager = new MigrationManager(this.tenantCache);
+
     // Load and upsert the persisted queries.
     this.persistedQueryCache = new PersistedQueryCache({ mongo: this.mongo });
 
@@ -188,13 +195,11 @@ class Server {
     }
     this.processing = true;
 
-    // Create the database indexes if it isn't disabled.
-    if (!this.config.get("disable_mongodb_autoindexing")) {
-      // Setup the database indexes.
-      logger.info("mongodb autoindexing is enabled, starting indexing");
-      await ensureIndexes(this.mongo);
+    // Run migrations if there is already a Tenant installed.
+    if (await isInstalled(this.tenantCache)) {
+      await this.migrationManager.executePendingMigrations(this.mongo);
     } else {
-      logger.warn("mongodb autoindexing is disabled, skipping indexing");
+      logger.info("no tenants are installed, skipping running migrations");
     }
 
     // Prime the queries in the database.
@@ -310,6 +315,7 @@ class Server {
       persistedQueriesRequired:
         this.config.get("env") === "production" &&
         !this.config.get("enable_graphiql"),
+      migrationManager: this.migrationManager,
     };
 
     // Only enable the metrics server if concurrency is set to 1.

src/core/server/logger/serializers.ts

@@ -3,13 +3,14 @@ import { GraphQLError } from "graphql";
 import StackUtils from "stack-utils";
 
 import { CoralError, CoralErrorContext } from "coral-server/errors";
+import VError from "verror";
 
 interface SerializedError {
   id?: string;
   message: string;
   name: string;
   stack?: string;
-  context?: CoralErrorContext;
+  context?: CoralErrorContext | Record<string, any>;
   originalError?: SerializedError;
 }
 
@@ -39,6 +40,8 @@ const errSerializer = (err: Error) => {
     if (cause) {
       obj.originalError = errSerializer(cause);
     }
+  } else if (err instanceof VError) {
+    obj.context = VError.info(err);
   }
 
   return obj;

src/core/server/models/action/comment.ts

@@ -17,16 +17,12 @@ import logger from "coral-server/logger";
 import {
   Connection,
   ConnectionInput,
-  createCollection,
-  createConnectionOrderVariants,
-  createIndexFactory,
   FilterQuery,
   Query,
   resolveConnection,
 } from "coral-server/models/helpers";
 import { TenantResource } from "coral-server/models/tenant";
-
-const collection = createCollection<CommentAction>("commentActions");
+import { commentActions as collection } from "coral-server/services/mongodb/collections";
 
 export enum ACTION_TYPE {
   /**
@@ -134,32 +130,6 @@ export interface CommentAction extends TenantResource {
   metadata?: Record<string, any>;
 }
 
-export async function createCommentActionIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  // { actionType, commentID, userID }
-  await createIndex(
-    { tenantID: 1, actionType: 1, commentID: 1, userID: 1 },
-    { background: true }
-  );
-
-  const variants = createConnectionOrderVariants<Readonly<CommentAction>>(
-    [{ createdAt: -1 }],
-    { background: true }
-  );
-
-  // Connection pagination.
-  // { ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    actionType: 1,
-    commentID: 1,
-  });
-}
-
 const ActionSchema = [
   // Flags
   {

src/core/server/models/action/moderation/comment.ts

@@ -6,33 +6,11 @@ import { GQLCOMMENT_STATUS } from "coral-server/graph/tenant/schema/__generated_
 import {
   Connection,
   ConnectionInput,
-  createCollection,
-  createConnectionOrderVariants,
-  createIndexFactory,
   Query,
   resolveConnection,
 } from "coral-server/models/helpers";
 import { TenantResource } from "coral-server/models/tenant";
-
-const collection = createCollection<CommentModerationAction>(
-  "commentModerationActions"
-);
-
-export async function createCommentModerationActionIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  const createVariants = createConnectionOrderVariants<
-    Readonly<CommentModerationAction>
-  >([{ createdAt: -1 }]);
-
-  // { moderatorID, ...connectionParams }
-  await createVariants(createIndex, {
-    moderatorID: 1,
-  });
-}
+import { commentModerationActions as collection } from "coral-server/services/mongodb/collections";
 
 /**
  * CommentModerationAction stores information around a moderation action that

src/core/server/models/comment/comment.ts

@@ -19,10 +19,7 @@ import {
 } from "coral-server/models/action/comment";
 import {
   Connection,
-  createCollection,
   createConnection,
-  createConnectionOrderVariants,
-  createIndexFactory,
   doesNotContainNull,
   FilterQuery,
   nodesToEdges,
@@ -32,6 +29,7 @@ import {
   resolveConnection,
 } from "coral-server/models/helpers";
 import { TenantResource } from "coral-server/models/tenant";
+import { comments as collection } from "coral-server/services/mongodb/collections";
 
 import { PUBLISHED_STATUSES } from "./constants";
 import {
@@ -42,8 +40,6 @@ import {
 import { Revision } from "./revision";
 import { CommentTag } from "./tag";
 
-const collection = createCollection<Comment>("comments");
-
 /**
  * Comment's are created by User's on Stories. Each Comment contains a body, and
  * can be moderated by another Moderator or Admin User.
@@ -128,84 +124,6 @@ export interface Comment extends TenantResource {
   deletedAt?: Date;
 }
 
-export async function createCommentIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  // Facility for counting the tags against a story.
-  await createIndex(
-    {
-      tenantID: 1,
-      storyID: 1,
-      "tags.type": 1,
-      status: 1,
-    },
-    {
-      background: true,
-      partialFilterExpression: {
-        "tags.type": { $exists: true },
-      },
-    }
-  );
-
-  const variants = createConnectionOrderVariants<Readonly<Comment>>([
-    { createdAt: -1 },
-    { createdAt: 1 },
-    { childCount: -1, createdAt: -1 },
-    { "actionCounts.REACTION": -1, createdAt: -1 },
-  ]);
-
-  // Story based Comment Connection pagination.
-  // { storyID, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    storyID: 1,
-    status: 1,
-  });
-
-  // Moderation based Comment Connection pagination.
-  // { storyID, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    status: 1,
-  });
-
-  // Story based Comment Connection pagination that are flagged.
-  // { storyID, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    storyID: 1,
-    status: 1,
-    "actionCounts.FLAG": 1,
-  });
-
-  // Story + Reply based Comment Connection pagination.
-  // { storyID, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    storyID: 1,
-    parentID: 1,
-    status: 1,
-  });
-
-  // Author based Comment Connection pagination.
-  // { authorID, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    authorID: 1,
-    status: 1,
-  });
-
-  // Tag based Comment Connection pagination.
-  // { tags.type, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    "tags.type": 1,
-  });
-}
-
 export type CreateCommentInput = Omit<
   Comment,
   | "id"

src/core/server/models/helpers/index.ts

@@ -1,5 +1,4 @@
 export * from "./collection";
 export * from "./connection";
-export * from "./indexing";
 export { default as Query } from "./query";
 export * from "./query";

src/core/server/models/invite.ts

@@ -3,23 +3,8 @@ import uuid from "uuid";
 
 import { Omit, Sub } from "coral-common/types";
 import { GQLUSER_ROLE } from "coral-server/graph/tenant/schema/__generated__/types";
-import {
-  createCollection,
-  createIndexFactory,
-} from "coral-server/models/helpers";
 import { TenantResource } from "coral-server/models/tenant";
-
-const collection = createCollection<Readonly<Invite>>("invites");
-
-export async function createInviteIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  // UNIQUE { email }
-  await createIndex({ tenantID: 1, email: 1 }, { unique: true });
-}
+import { invites as collection } from "coral-server/services/mongodb/collections";
 
 export interface Invite extends TenantResource {
   readonly id: string;

src/core/server/models/migration/index.ts

@@ -0,0 +1 @@
+export * from "./migration";

src/core/server/models/migration/migration.ts

@@ -0,0 +1,96 @@
+import { Db } from "mongodb";
+
+import { migrations as collection } from "coral-server/services/mongodb/collections";
+
+export enum MIGRATION_STATE {
+  STARTED = "STARTED",
+  FAILED = "FAILED",
+  FINISHED = "FINISHED",
+}
+
+export interface MigrationRecord {
+  id: number;
+  state: MIGRATION_STATE;
+  clientID?: string;
+  createdAt: Date;
+  updatedAt?: Date;
+}
+
+export async function startMigration(
+  mongo: Db,
+  id: number,
+  clientID: string,
+  now = new Date()
+) {
+  const result = await collection(mongo).findOneAndUpdate(
+    { id },
+    {
+      $setOnInsert: {
+        id,
+        clientID,
+        state: MIGRATION_STATE.STARTED,
+        createdAt: now,
+      },
+    },
+    {
+      // False to return the updated document instead of the original document.
+      returnOriginal: false,
+      upsert: true,
+    }
+  );
+  if (!result.value) {
+    throw new Error("an unexpected error occurred");
+  }
+
+  return result.value;
+}
+
+/**
+ * updateMigrationState will update the state of a migration record to reflect
+ * the new state as well as un-setting the client ID from the records.
+ *
+ * @param mongo the database to interact on
+ * @param id the migration version to update
+ * @param state the state to switch the record to
+ * @param now the current date
+ */
+async function updateMigrationState(
+  mongo: Db,
+  id: number,
+  state: MIGRATION_STATE.FINISHED | MIGRATION_STATE.FAILED,
+  now: Date
+) {
+  const result = await collection(mongo).findOneAndUpdate(
+    { id },
+    {
+      $set: {
+        state,
+        updatedAt: now,
+      },
+      $unset: {
+        clientID: "",
+      },
+    },
+    {
+      // False to return the updated document instead of the original document.
+      returnOriginal: false,
+    }
+  );
+
+  return result.value || null;
+}
+
+export async function finishMigration(mongo: Db, id: number, now = new Date()) {
+  return updateMigrationState(mongo, id, MIGRATION_STATE.FINISHED, now);
+}
+
+export async function failMigration(mongo: Db, id: number, now = new Date()) {
+  return updateMigrationState(mongo, id, MIGRATION_STATE.FAILED, now);
+}
+
+export async function retrieveAllMigrationRecords(mongo: Db) {
+  const cursor = await collection(mongo)
+    .find({})
+    .sort({ id: 1 });
+  return cursor.toArray();
+}

src/core/server/models/queries/queries.ts

@@ -3,12 +3,7 @@ import { Db, MongoError } from "mongodb";
 
 import { waitFor } from "coral-common/helpers";
 import logger from "coral-server/logger";
-import {
-  createCollection,
-  createIndexFactory,
-} from "coral-server/models/helpers";
-
-const collection = createCollection<Readonly<PersistedQuery>>("queries");
+import { queries as collection } from "coral-server/services/mongodb/collections";
 
 export interface PersistedQuery {
   id: string;
@@ -19,13 +14,6 @@ export interface PersistedQuery {
   version: string;
 }
 
-export async function createQueriesIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ id: 1 }, { unique: true });
-}
-
 export async function primeQueries(
   mongo: Db,
   queries: PersistedQuery[],

src/core/server/models/story/counts/index.ts

@@ -13,28 +13,12 @@ import {
   CommentStatusCounts,
   createEmptyCommentStatusCounts,
 } from "coral-server/models/comment/helpers";
-import {
-  createCollection,
-  createIndexFactory,
-} from "coral-server/models/helpers";
 import { retrieveStory, Story } from "coral-server/models/story";
+import { stories as collection } from "coral-server/services/mongodb/collections";
 import { AugmentedRedis } from "coral-server/services/redis";
 
 import { updateSharedCommentCounts } from "./shared";
 
-/**
- * collection provides a reference to the stories collection used by the
- * counting system.
- */
-const collection = createCollection<Story>("stories");
-
-export async function createStoryCountIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // { createdAt }
-  await createIndex({ tenantID: 1, createdAt: 1 }, { background: true });
-}
-
 /**
  * CommentModerationCountsPerQueue stores the number of Comments that exist in
  * each of the Moderation Queues.

src/core/server/models/story/counts/shared.ts

@@ -8,12 +8,11 @@ import {
   CommentStatusCounts,
   createEmptyCommentStatusCounts,
 } from "coral-server/models/comment/helpers";
-import { createCollection } from "coral-server/models/helpers";
-import { Story } from "coral-server/models/story";
 import {
   CommentModerationCountsPerQueue,
   StoryCounts,
 } from "coral-server/models/story/counts";
+import { stories as collection } from "coral-server/services/mongodb/collections";
 import { AugmentedPipeline, AugmentedRedis } from "coral-server/services/redis";
 
 import {
@@ -46,12 +45,6 @@ const commentCountsModerationQueueTotalKey = (tenantID: string) =>
 const commentCountsModerationQueueQueuesKey = (tenantID: string) =>
   `${tenantID}:commentCounts:moderationQueue:queues`;
 
-/**
- * collection provides a reference to the stories collection used by the
- * counting system.
- */
-const collection = createCollection<Story>("stories");
-
 /**
  * recalculateSharedModerationQueueQueueCounts will reset the counts stored for
  * this Tenant.

src/core/server/models/story/index.ts

@@ -14,16 +14,14 @@ import {
 import {
   Connection,
   ConnectionInput,
-  createConnectionOrderVariants,
-  createIndexFactory,
   Query,
   resolveConnection,
 } from "coral-server/models/helpers";
 import { GlobalModerationSettings } from "coral-server/models/settings";
 import { TenantResource } from "coral-server/models/tenant";
+import { stories as collection } from "coral-server/services/mongodb/collections";
 
 import { createEmptyCommentStatusCounts } from "../comment/helpers";
-import { createCollection } from "../helpers/collection";
 import {
   createEmptyCommentModerationQueueCounts,
   StoryCommentCounts,
@@ -32,8 +30,6 @@ import {
 export * from "./counts";
 export * from "./helpers";
 
-const collection = createCollection<Story>("stories");
-
 export type StorySettings = DeepPartial<
   Pick<GQLStorySettings, "messageBox"> & GlobalModerationSettings
 >;
@@ -81,40 +77,6 @@ export interface Story extends TenantResource {
   createdAt: Date;
 }
 
-export async function createStoryIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  // UNIQUE { url }
-  await createIndex({ tenantID: 1, url: 1 }, { unique: true });
-
-  // TEXT { $**, createdAt }
-  await createIndex(
-    { tenantID: 1, "$**": "text", createdAt: -1 },
-    { background: true }
-  );
-
-  const variants = createConnectionOrderVariants<Readonly<Story>>(
-    [{ createdAt: -1 }],
-    { background: true }
-  );
-
-  // Story Connection pagination.
-  // { ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-  });
-
-  // Closed At ordered Story Connection pagination.
-  // { closedAt, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    closedAt: 1,
-  });
-}
-
 export interface UpsertStoryInput {
   id?: string;
   url: string;

src/core/server/models/tenant/tenant.ts

@@ -9,15 +9,11 @@ import {
   GQLMODERATION_MODE,
   GQLSettings,
 } from "coral-server/graph/tenant/schema/__generated__/types";
-import {
-  createCollection,
-  createIndexFactory,
-} from "coral-server/models/helpers";
 import { Settings } from "coral-server/models/settings";
 import { I18n } from "coral-server/services/i18n";
-import { generateSSOKey, getDefaultReactionConfiguration } from "./helpers";
+import { tenants as collection } from "coral-server/services/mongodb/collections";
 
-const collection = createCollection<Tenant>("tenants");
+import { generateSSOKey, getDefaultReactionConfiguration } from "./helpers";
 
 /**
  * TenantResource references a given resource that should be owned by a specific
@@ -46,16 +42,6 @@ export interface TenantSettings
  */
 export type Tenant = Settings & TenantSettings;
 
-export async function createTenantIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ id: 1 }, { unique: true });
-
-  // UNIQUE { domain }
-  await createIndex({ domain: 1 }, { unique: true });
-}
-
 /**
  * CreateTenantInput is the set of properties that can be set when a given
  * Tenant is created. The remainder of the properties are set from defaults and

src/core/server/models/user/user.ts

@@ -33,19 +33,15 @@ import logger from "coral-server/logger";
 import {
   Connection,
   ConnectionInput,
-  createCollection,
-  createConnectionOrderVariants,
-  createIndexFactory,
   Query,
   resolveConnection,
 } from "coral-server/models/helpers";
 import { TenantResource } from "coral-server/models/tenant";
 import { DigestibleTemplate } from "coral-server/queue/tasks/mailer/templates";
+import { users as collection } from "coral-server/services/mongodb/collections";
 
 import { getLocalProfile, hasLocalProfile } from "./helpers";
 
-const collection = createCollection<User>("users");
-
 export interface LocalProfile {
   type: "local";
   id: string;
@@ -297,10 +293,8 @@ export interface UserStatus {
   /**
    * premod stores whether a user is set to mandatory premod and history of
    * premod status.
-   *
-   * FIXME: (wyattjoh) set defaults during migration
    */
-  premod?: PremodStatus;
+  premod: PremodStatus;
 }
 
 /**
@@ -436,80 +430,6 @@ export interface User extends TenantResource {
   deletedAt?: Date;
 }
 
-export async function createUserIndexes(mongo: Db) {
-  const createIndex = createIndexFactory(collection(mongo));
-
-  // UNIQUE { id }
-  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
-
-  // UNIQUE - PARTIAL { email }
-  await createIndex(
-    { tenantID: 1, email: 1 },
-    { unique: true, partialFilterExpression: { email: { $exists: true } } }
-  );
-
-  // UNIQUE { profiles.type, profiles.id }
-  await createIndex(
-    { tenantID: 1, "profiles.type": 1, "profiles.id": 1 },
-    {
-      unique: true,
-      partialFilterExpression: { "profiles.id": { $exists: true } },
-    }
-  );
-
-  // { profiles }
-  await createIndex(
-    { tenantID: 1, profiles: 1, email: 1 },
-    {
-      partialFilterExpression: { profiles: { $exists: true } },
-      background: true,
-    }
-  );
-
-  // TEXT { id, username, email, createdAt }
-  await createIndex(
-    {
-      tenantID: 1,
-      id: "text",
-      username: "text",
-      email: "text",
-      createdAt: -1,
-    },
-    { background: true }
-  );
-
-  const variants = createConnectionOrderVariants<Readonly<User>>(
-    [{ createdAt: -1 }],
-    { background: true }
-  );
-
-  // User Connection pagination.
-  // { ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-  });
-
-  // Role based User Connection pagination.
-  // { role, ...connectionParams }
-  await variants(createIndex, {
-    tenantID: 1,
-    role: 1,
-  });
-
-  // Suspension based User Connection pagination.
-  await variants(createIndex, {
-    tenantID: 1,
-    "status.suspension.history.from.start": 1,
-    "status.suspension.history.from.finish": 1,
-  });
-
-  // Ban based User Connection pagination.
-  await variants(createIndex, {
-    tenantID: 1,
-    "status.ban.active": 1,
-  });
-}
-
 function hashPassword(password: string): Promise<string> {
   return bcrypt.hash(password, 10);
 }
@@ -1460,8 +1380,7 @@ export async function premodUser(
 
     // Check to see if the user is already banned.
     const premod = consolidateUserPremodStatus(user.status.premod);
-    // FIXME: (wyattjoh) once migration has been performed, remove check
-    if (premod && premod.active) {
+    if (premod.active) {
       throw new UserAlreadyPremoderated();
     }
 
@@ -1873,8 +1792,7 @@ export function consolidateUserSuspensionStatus(
 export interface ConsolidatedUserStatus {
   suspension: ConsolidatedSuspensionStatus;
   ban: ConsolidatedBanStatus;
-  // FIXME: (wyattjoh) once migration has been performed, make required
-  premod?: ConsolidatedPremodStatus;
+  premod: ConsolidatedPremodStatus;
 }
 
 export function consolidateUserStatus(

src/core/server/services/comments/pipeline/phases/preModerateUser.ts

@@ -8,8 +8,7 @@ import {
 export const premodUser: IntermediateModerationPhase = ({
   author,
 }): IntermediatePhaseResult | void => {
-  // FIXME: (wyattjoh) once migration has been performed, remove check
-  if (author.status.premod && author.status.premod.active) {
+  if (author.status.premod.active) {
     return {
       status: GQLCOMMENT_STATUS.PREMOD,
     };

src/core/server/services/migrate/error.ts

@@ -0,0 +1,63 @@
+// tslint:disable: max-classes-per-file
+
+import VError from "verror";
+
+import { MigrationRecord } from "coral-server/models/migration";
+
+export class MigrationError extends VError {
+  public readonly tenantID: string;
+  public readonly reason: string;
+  public readonly affectedCollection?: string;
+  public readonly affectedIDs?: string[];
+
+  constructor(
+    tenantID: string,
+    reason: string,
+    affectedCollection?: string,
+    affectedIDs?: string[]
+  ) {
+    super(
+      {
+        name: "MigrationError",
+        info: {
+          tenantID,
+          affectedCollection,
+          affectedIDs,
+        },
+      },
+      'MigrationError: "%s"',
+      reason
+    );
+
+    this.tenantID = tenantID;
+    this.reason = reason;
+    this.affectedCollection = affectedCollection;
+    this.affectedIDs = affectedIDs;
+  }
+}
+
+export class FailedMigrationDetectedError extends VError {
+  constructor(record: MigrationRecord) {
+    super(
+      {
+        name: "FailedMigrationDetectedError",
+        info: record,
+      },
+      'FailedMigrationDetectedError: migration "%d" failed, remove this document to restart the migration process',
+      record.id
+    );
+  }
+}
+
+export class InProgressMigrationDetectedError extends VError {
+  constructor(record: MigrationRecord) {
+    super(
+      {
+        name: "InProgressMigrationDetectedError",
+        info: record,
+      },
+      'InProgressMigrationDetectedError: migration "%d" was in progress',
+      record.id
+    );
+  }
+}

src/core/server/services/migrate/index.ts

@@ -0,0 +1,2 @@
+export { default as Migration } from "./migration";
+export { default as MigrationManager } from "./manager";

src/core/server/services/migrate/indexing.ts

@@ -1,5 +1,6 @@
 import { merge } from "lodash";
 import { Collection, IndexOptions } from "mongodb";
+import now from "performance-now";
 
 import { Writable } from "coral-common/types";
 import logger from "coral-server/logger";
@@ -32,8 +33,13 @@ export function createIndexFactory<T>(
   ) => {
     try {
       // Try to create the index.
+      const start = now();
+      log.debug({ indexSpec, indexOptions }, "creating index");
       const indexName = await collection.createIndex(indexSpec, indexOptions);
-      log.debug({ indexName, indexSpec, indexOptions }, "index was created");
+      log.debug(
+        { indexName, indexSpec, indexOptions, took: Math.round(now() - start) },
+        "index was created"
+      );
 
       // Match the interface from the `createIndex` function by returning the
       // index name.
@@ -48,11 +54,11 @@ export function createIndexFactory<T>(
 }
 
 export function createConnectionOrderVariants<T>(
+  createIndex: IndexCreationFunction<T>,
   variants: Array<IndexSpecification<T>>,
-  indexOptions: IndexOptions = {}
+  indexOptions: IndexOptions = { background: true }
 ) {
   return async (
-    createIndex: IndexCreationFunction<T>,
     indexSpec: IndexSpecification<T>,
     variantIndexOptions: IndexOptions = {}
   ) => {
@@ -68,9 +74,6 @@ export function createConnectionOrderVariants<T>(
         merge({}, indexOptions, variantIndexOptions)
       );
 
-    // Create a raw index without the variants applied.
-    await createIndex(indexSpec, merge({}, indexOptions, variantIndexOptions));
-
     // Create all the variants.
     for (const variant of variants) {
       await createIndexVariant(variant);

src/core/server/services/migrate/manager.ts

@@ -0,0 +1,238 @@
+import fs from "fs-extra";
+import { Db } from "mongodb";
+import path from "path";
+import now from "performance-now";
+import uuid from "uuid";
+
+import logger from "coral-server/logger";
+import {
+  failMigration,
+  finishMigration,
+  MIGRATION_STATE,
+  retrieveAllMigrationRecords,
+  startMigration,
+} from "coral-server/models/migration";
+import TenantCache from "coral-server/services/tenant/cache";
+
+import {
+  FailedMigrationDetectedError,
+  InProgressMigrationDetectedError,
+} from "./error";
+import Migration from "./migration";
+
+// Extract the id from the filename with this regex.
+const fileNamePattern = /^(\d+)_([\S_]+)\.[tj]s$/;
+
+export default class Manager {
+  private clientID: string;
+  private migrations: Migration[];
+  private tenants: TenantCache;
+  private ran: boolean = false;
+
+  constructor(tenants: TenantCache) {
+    this.clientID = uuid.v4();
+    this.migrations = [];
+    this.tenants = tenants;
+
+    const fileNames = fs.readdirSync(path.join(__dirname, "migrations"));
+    for (const fileName of fileNames) {
+      // Test to see if this fileName is one of our migrations.
+      if (!fileNamePattern.test(fileName)) {
+        if (fileName.endsWith(".map")) {
+          // This was a mapping file (in production), skip this.
+          continue;
+        }
+
+        logger.warn(
+          { fileName },
+          "found a file in the migrations folder that did not have an expected format"
+        );
+        continue;
+      }
+
+      // Load the migration.
+      const filePath = path.join(__dirname, "migrations", fileName);
+      const m = require(filePath);
+
+      // Parse the timestamp out of the migration filename.
+      const matches = fileName.match(fileNamePattern);
+      if (!matches || matches.length !== 3) {
+        throw new Error("fileName format is invalid");
+      }
+      const id = parseInt(matches[1], 10);
+
+      // Create the migration instance.
+      const migration = new m.default(id, matches[2]);
+
+      // Insert the migration into the migrations array.
+      if (!(migration instanceof Migration)) {
+        throw new Error(`migration at ${filePath} did not export a Migration`);
+      }
+
+      this.migrations.push(migration);
+    }
+
+    // Sort the migrations.
+    this.migrations.sort((a, b) => {
+      if (a.id < b.id) {
+        return -1;
+      }
+
+      if (a.id > b.id) {
+        return 1;
+      }
+
+      return 0;
+    });
+  }
+
+  /**
+   * pending will return the pending migrations that need to be completed.
+   *
+   * @param mongo the database handle to use to get the migrations
+   */
+  private async pending(mongo: Db): Promise<Migration[]> {
+    // Get all the migration records in the database.
+    const records = await retrieveAllMigrationRecords(mongo);
+
+    // Check to see if any of the migrations have failed or are in progress.
+    for (const record of records) {
+      switch (record.state) {
+        case MIGRATION_STATE.FAILED:
+          throw new FailedMigrationDetectedError(record);
+        case MIGRATION_STATE.STARTED:
+          throw new InProgressMigrationDetectedError(record);
+        default:
+          break;
+      }
+    }
+
+    return this.migrations.filter(migration => {
+      // Find the record based on the migration.
+      const record = records.find(({ id }) => migration.id === id);
+      if (record) {
+        // The record exists, so it isn't pending, it's already finished.
+        return false;
+      }
+
+      // A record of the migration does not exist, so mark it as pending.
+      return true;
+    });
+  }
+
+  private async currentMigration(mongo: Db) {
+    const records = await retrieveAllMigrationRecords(mongo);
+    return records.length > 0 ? records[records.length - 1] : null;
+  }
+
+  public async executePendingMigrations(mongo: Db) {
+    // Error out if this is ran twice.
+    if (this.ran) {
+      throw new Error("pending migrations have already been executed");
+    }
+
+    // Mark the migrations as ran.
+    this.ran = true;
+
+    // Check the current migration id.
+    let currentMigration = await this.currentMigration(mongo);
+
+    // Determine which migrations need to be ran.
+    const pending = await this.pending(mongo);
+    if (pending.length === 0) {
+      logger.info(
+        {
+          currentMigrationID: currentMigration ? currentMigration.id : null,
+        },
+        "there was no pending migrations to run"
+      );
+      return;
+    }
+
+    logger.info({ pending: pending.length }, "executing pending migrations");
+
+    const migrationsStartTime = now();
+
+    for (const migration of pending) {
+      let log = logger.child(
+        {
+          migrationName: migration.name,
+          migrationID: migration.id,
+        },
+        true
+      );
+
+      // Mark the migration as started.
+      const record = await startMigration(mongo, migration.id, this.clientID);
+      if (record.clientID !== this.clientID) {
+        throw new InProgressMigrationDetectedError(record);
+      }
+
+      // Apply any index changes for the migration.
+      if (migration.indexes) {
+        const migrationStartTime = now();
+        log.info("starting index migration");
+        await migration.indexes(mongo);
+        const executionTime = Math.round(now() - migrationStartTime);
+        log.info({ executionTime }, "finished index migration");
+      }
+
+      if (migration.up) {
+        // The migration provides an up method, we should run this per Tenant.
+        for await (const tenant of this.tenants) {
+          log = log.child({ tenantID: tenant.id }, true);
+
+          const migrationStartTime = now();
+          log.info("starting migration");
+
+          try {
+            // Up the migration.
+            await migration.up(mongo, tenant.id);
+
+            // Test the migration.
+            if (migration.test) {
+              await migration.test(mongo, tenant.id);
+            }
+          } catch (err) {
+            // The migration or test has failed, try to roll back the operation.
+            if (migration.down) {
+              log.error({ err }, "migration has failed, attempting rollback");
+
+              // Attempt the down migration.
+              await migration.down(mongo, tenant.id);
+            } else {
+              log.error(
+                { err },
+                "migration has failed, and does not have a down method available, migration will not be rolled back"
+              );
+            }
+
+            // Mark the migration as failed.
+            await failMigration(mongo, migration.id);
+
+            // Rethrow the error here to cause the application to crash.
+            throw err;
+          }
+
+          const executionTime = Math.round(now() - migrationStartTime);
+          log.info({ executionTime }, "finished migration");
+        }
+      }
+
+      // Mark the migration as completed.
+      await finishMigration(mongo, migration.id);
+    }
+
+    const finishTime = Math.round(now() - migrationsStartTime);
+
+    currentMigration = await this.currentMigration(mongo);
+
+    logger.info(
+      {
+        finishTime,
+        currentMigrationID: currentMigration ? currentMigration.id : null,
+      },
+      "finished running pending migrations"
+    );
+  }
+}

src/core/server/services/migrate/migration.ts

@@ -0,0 +1,35 @@
+import Logger from "bunyan";
+import { Db } from "mongodb";
+
+import logger from "coral-server/logger";
+
+interface Migration {
+  indexes?(mongo: Db): Promise<void>;
+  up?(mongo: Db, tenantID: string): Promise<void>;
+  test?(mongo: Db, tenantID: string): Promise<void>;
+  down?(mongo: Db, tenantID: string): Promise<void>;
+}
+
+abstract class Migration {
+  public readonly name: string;
+  public readonly id: number;
+  public readonly logger: Logger;
+
+  constructor(version: number, name: string) {
+    this.id = version;
+    this.name = name;
+    this.logger = logger.child(
+      {
+        migrationName: this.name,
+        migrationVersion: this.id,
+      },
+      true
+    );
+  }
+
+  protected log(tenantID: string) {
+    return this.logger.child({ tenantID }, true);
+  }
+}
+
+export default Migration;

src/core/server/services/migrate/migration_sample.ts

@@ -0,0 +1,13 @@
+import { Db } from "mongodb";
+
+// Use the following collections reference to interact with specific
+// collections.
+// import collections from "coral-server/services/mongodb/collections";
+
+import Migration from "coral-server/services/migrate/migration";
+
+export default class extends Migration {
+  public async up(mongo: Db, tenantID: string) {
+    throw new Error("migration not implemented");
+  }
+}

src/core/server/services/migrate/migrations/1569455150152_premod_user_status.ts

@@ -0,0 +1,77 @@
+import { Db } from "mongodb";
+
+import collections from "coral-server/services/mongodb/collections";
+
+import { MigrationError } from "../error";
+import Migration from "../migration";
+
+export default class extends Migration {
+  public async down(mongo: Db, tenantID: string) {
+    // Remove the premod user status from all users on this Tenant.
+    const result = await collections.users(mongo).updateMany(
+      {
+        tenantID,
+        "status.premod": { $ne: null },
+      },
+      {
+        $unset: {
+          "status.premod": "",
+        },
+      }
+    );
+
+    this.log(tenantID).warn(
+      { matchedCount: result.matchedCount, modifiedCount: result.matchedCount },
+      "cleared the premod status from users"
+    );
+  }
+
+  public async test(mongo: Db, tenantID: string) {
+    // Find all the users that still have premod status unset.
+    const cursor = await collections
+      .users(mongo)
+      .find({
+        "status.premod": null,
+        tenantID,
+      })
+      .project({ id: 1 });
+
+    // Count them.
+    const users = await cursor.toArray();
+    const count = users.length;
+    if (count === 0) {
+      this.log(tenantID).info("all users migrated successfully");
+      return;
+    }
+
+    throw new MigrationError(
+      tenantID,
+      "found users that were not updated",
+      "users",
+      users.map(({ id }) => id)
+    );
+  }
+
+  public async up(mongo: Db, tenantID: string) {
+    // Migrate users to include the premod status.
+    const result = await collections.users(mongo).updateMany(
+      {
+        "status.premod": null,
+        tenantID,
+      },
+      {
+        $set: {
+          "status.premod": {
+            active: false,
+            history: [],
+          },
+        },
+      }
+    );
+
+    this.log(tenantID).info(
+      { matchedCount: result.matchedCount, modifiedCount: result.matchedCount },
+      "updated user premod status"
+    );
+  }
+}

src/core/server/services/migrate/migrations/1569612830133_indexes.ts

@@ -0,0 +1,342 @@
+import { Db } from "mongodb";
+
+import Migration from "coral-server/services/migrate/migration";
+import collections from "coral-server/services/mongodb/collections";
+
+import { createConnectionOrderVariants, createIndexFactory } from "../indexing";
+
+async function createMigrationRecordIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.migrations(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ id: 1 }, { unique: true });
+}
+
+async function createUserIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.users(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  // UNIQUE - PARTIAL { email }
+  await createIndex(
+    { tenantID: 1, email: 1 },
+    { unique: true, partialFilterExpression: { email: { $exists: true } } }
+  );
+
+  // UNIQUE { profiles.type, profiles.id }
+  await createIndex(
+    { tenantID: 1, "profiles.type": 1, "profiles.id": 1 },
+    {
+      unique: true,
+      partialFilterExpression: { "profiles.id": { $exists: true } },
+    }
+  );
+
+  // { profiles }
+  await createIndex(
+    { tenantID: 1, profiles: 1, email: 1 },
+    {
+      partialFilterExpression: { profiles: { $exists: true } },
+      background: true,
+    }
+  );
+
+  // TEXT { id, username, email, createdAt }
+  await createIndex(
+    {
+      tenantID: 1,
+      id: "text",
+      username: "text",
+      email: "text",
+      createdAt: -1,
+    },
+    { background: true }
+  );
+
+  const variants = createConnectionOrderVariants(
+    createIndex,
+    [{ createdAt: -1 }],
+    { background: true }
+  );
+
+  // User Connection pagination.
+  // { ...connectionParams }
+  await variants({
+    tenantID: 1,
+  });
+
+  // Role based User Connection pagination.
+  // { role, ...connectionParams }
+  await variants({
+    tenantID: 1,
+    role: 1,
+  });
+
+  // Suspension based User Connection pagination.
+  await variants({
+    tenantID: 1,
+    "status.suspension.history.from.start": 1,
+    "status.suspension.history.from.finish": 1,
+  });
+
+  // Ban based User Connection pagination.
+  await variants({
+    tenantID: 1,
+    "status.ban.active": 1,
+  });
+}
+
+async function createInviteIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.invites(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  // UNIQUE { email }
+  await createIndex({ tenantID: 1, email: 1 }, { unique: true });
+}
+
+async function createTenantIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.tenants(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ id: 1 }, { unique: true });
+
+  // UNIQUE { domain }
+  await createIndex({ domain: 1 }, { unique: true });
+}
+
+async function createStoryIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.stories(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  // UNIQUE { url }
+  await createIndex({ tenantID: 1, url: 1 }, { unique: true });
+
+  // TEXT { $**, createdAt }
+  await createIndex(
+    { tenantID: 1, "$**": "text", createdAt: -1 },
+    { background: true }
+  );
+
+  const variants = createConnectionOrderVariants(
+    createIndex,
+    [{ createdAt: -1 }],
+    { background: true }
+  );
+
+  // Story Connection pagination.
+  // { ...connectionParams }
+  await variants({
+    tenantID: 1,
+  });
+
+  // Closed At ordered Story Connection pagination.
+  // { closedAt, ...connectionParams }
+  await variants({
+    tenantID: 1,
+    closedAt: 1,
+  });
+}
+
+async function createStoryCountIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.stories(mongo));
+
+  // { createdAt }
+  await createIndex({ tenantID: 1, createdAt: 1 }, { background: true });
+}
+
+async function createQueriesIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.queries(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ id: 1 }, { unique: true });
+}
+
+async function createCommentActionIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.commentActions(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  // { actionType, commentID, userID }
+  await createIndex(
+    { tenantID: 1, actionType: 1, commentID: 1, userID: 1 },
+    { background: true }
+  );
+
+  const variants = createConnectionOrderVariants(
+    createIndex,
+    [{ createdAt: -1 }],
+    { background: true }
+  );
+
+  // Connection pagination.
+  // { ...connectionParams }
+  await variants({
+    tenantID: 1,
+    actionType: 1,
+    commentID: 1,
+  });
+}
+
+async function createCommentModerationActionIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(
+    collections.commentModerationActions(mongo)
+  );
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  const createVariants = createConnectionOrderVariants(createIndex, [
+    { createdAt: -1 },
+  ]);
+
+  // { moderatorID, ...connectionParams }
+  await createVariants({
+    moderatorID: 1,
+  });
+}
+
+async function createCommentIndexes(mongo: Db) {
+  const createIndex = createIndexFactory(collections.comments(mongo));
+
+  // UNIQUE { id }
+  await createIndex({ tenantID: 1, id: 1 }, { unique: true });
+
+  // Facility for counting the tags against a story.
+  await createIndex(
+    {
+      tenantID: 1,
+      storyID: 1,
+      "tags.type": 1,
+      status: 1,
+    },
+    {
+      background: true,
+      partialFilterExpression: {
+        "tags.type": { $exists: true },
+      },
+    }
+  );
+
+  const streamVariants = createConnectionOrderVariants(createIndex, [
+    { createdAt: -1 },
+    { createdAt: 1 },
+    { childCount: -1, createdAt: -1 },
+    { "actionCounts.REACTION": -1, createdAt: -1 },
+  ]);
+
+  // Story based Comment Connection pagination.
+  // { storyID, ...connectionParams }
+  await streamVariants({
+    tenantID: 1,
+    storyID: 1,
+    status: 1,
+  });
+
+  // Story + Reply based Comment Connection pagination.
+  // { storyID, ...connectionParams }
+  await streamVariants({
+    tenantID: 1,
+    storyID: 1,
+    parentID: 1,
+    status: 1,
+  });
+
+  // Author based Comment Connection pagination.
+  // { authorID, ...connectionParams }
+  await streamVariants({
+    tenantID: 1,
+    authorID: 1,
+    status: 1,
+  });
+
+  // Tag based Comment Connection pagination.
+  // { tags.type, ...connectionParams }
+  await streamVariants({
+    tenantID: 1,
+    storyID: 1,
+    "tags.type": 1,
+  });
+
+  const adminVariants = createConnectionOrderVariants(
+    createIndex,
+    [{ createdAt: -1 }, { createdAt: 1 }],
+    { background: true }
+  );
+
+  // Moderation based Comment Connection pagination.
+  // { storyID, ...connectionParams }
+  await adminVariants({
+    tenantID: 1,
+    status: 1,
+  });
+
+  // Story based Comment Connection pagination that are flagged.
+  // { storyID, ...connectionParams }
+  await adminVariants({
+    tenantID: 1,
+    storyID: 1,
+    status: 1,
+    "actionCounts.FLAG": 1,
+  });
+
+  // Author based Comment Connection pagination.
+  // { authorID, ...connectionParams }
+  await adminVariants({
+    tenantID: 1,
+    authorID: 1,
+  });
+}
+
+type IndexCreationFunction = (mongo: Db) => Promise<void>;
+
+const indexes: Array<[string, IndexCreationFunction]> = [
+  ["migrations", createMigrationRecordIndexes],
+  ["users", createUserIndexes],
+  ["invites", createInviteIndexes],
+  ["tenants", createTenantIndexes],
+  ["comments", createCommentIndexes],
+  ["stories", createStoryIndexes],
+  ["stories", createStoryCountIndexes],
+  ["commentActions", createCommentActionIndexes],
+  ["commentModerationActions", createCommentModerationActionIndexes],
+  ["queries", createQueriesIndexes],
+];
+
+export default class extends Migration {
+  /**
+   * ensureIndexes will ensure that all indexes have been created.
+   *
+   * @param mongo a MongoDB Database Connection
+   */
+  private async ensureIndexes(mongo: Db) {
+    this.logger.info(
+      { indexGroupCount: indexes.length },
+      "now ensuring indexes are created"
+    );
+
+    // For each of the index functions, call it.
+    for (const [indexGroup, indexFunction] of indexes) {
+      this.logger.info({ indexGroup }, "ensuring indexes are created");
+      await indexFunction(mongo);
+      this.logger.info({ indexGroup }, "indexes have been created");
+    }
+
+    this.logger.info("all indexes have been created");
+  }
+
+  public async indexes(mongo: Db) {
+    // Drop existing indexes on managed collections so we can re-create them.
+    for (const collection of Object.values(collections)) {
+      await collection(mongo).dropIndexes();
+    }
+
+    // Re-create the indexes for each collection now.
+    await this.ensureIndexes(mongo);
+  }
+}

src/core/server/services/mongodb/collections.ts

@@ -0,0 +1,45 @@
+import { createCollection } from "coral-server/models/helpers";
+
+import { CommentAction } from "coral-server/models/action/comment";
+import { CommentModerationAction } from "coral-server/models/action/moderation/comment";
+import { Comment } from "coral-server/models/comment";
+import { Invite } from "coral-server/models/invite";
+import { MigrationRecord } from "coral-server/models/migration";
+import { PersistedQuery } from "coral-server/models/queries";
+import { Story } from "coral-server/models/story";
+import { Tenant } from "coral-server/models/tenant";
+import { User } from "coral-server/models/user";
+
+export const users = createCollection<User>("users");
+
+export const invites = createCollection<Invite>("invites");
+
+export const tenants = createCollection<Tenant>("tenants");
+
+export const comments = createCollection<Comment>("comments");
+
+export const stories = createCollection<Story>("stories");
+
+export const commentActions = createCollection<CommentAction>("commentActions");
+
+export const commentModerationActions = createCollection<
+  CommentModerationAction
+>("commentModerationActions");
+
+export const queries = createCollection<PersistedQuery>("queries");
+
+export const migrations = createCollection<MigrationRecord>("migrations");
+
+const collections = {
+  users,
+  invites,
+  tenants,
+  comments,
+  stories,
+  commentActions,
+  commentModerationActions,
+  queries,
+  migrations,
+};
+
+export default collections;

src/core/server/services/mongodb/indexes.ts

@@ -1,49 +0,0 @@
-import { Db } from "mongodb";
-
-import logger from "coral-server/logger";
-import { createCommentActionIndexes } from "coral-server/models/action/comment";
-import { createCommentModerationActionIndexes } from "coral-server/models/action/moderation/comment";
-import { createCommentIndexes } from "coral-server/models/comment";
-import { createInviteIndexes } from "coral-server/models/invite";
-import { createQueriesIndexes } from "coral-server/models/queries";
-import {
-  createStoryCountIndexes,
-  createStoryIndexes,
-} from "coral-server/models/story";
-import { createTenantIndexes } from "coral-server/models/tenant";
-import { createUserIndexes } from "coral-server/models/user";
-
-type IndexCreationFunction = (mongo: Db) => Promise<void>;
-
-const indexes: Array<[string, IndexCreationFunction]> = [
-  ["users", createUserIndexes],
-  ["invites", createInviteIndexes],
-  ["tenants", createTenantIndexes],
-  ["comments", createCommentIndexes],
-  ["stories", createStoryIndexes],
-  ["stories", createStoryCountIndexes],
-  ["commentActions", createCommentActionIndexes],
-  ["commentModerationActions", createCommentModerationActionIndexes],
-  ["queries", createQueriesIndexes],
-];
-
-/**
- * ensureIndexes will ensure that all indexes have been created.
- *
- * @param mongo a MongoDB Database Connection
- */
-export async function ensureIndexes(mongo: Db) {
-  logger.debug(
-    { indexGroupCount: indexes.length },
-    "now ensuring indexes are created"
-  );
-
-  // For each of the index functions, call it.
-  for (const [indexGroup, indexFunction] of indexes) {
-    logger.debug({ indexGroup }, "ensuring indexes are created");
-    await indexFunction(mongo);
-    logger.debug({ indexGroup }, "indexes have been created");
-  }
-
-  logger.debug("all indexes have been created");
-}

src/core/server/services/tenant/cache/index.ts

@@ -198,6 +198,8 @@ export default class TenantCache {
 
         yield tenant;
       }
+
+      return;
     }
 
     // Caching must be disabled, so just grab all the tenants for this node and

src/core/server/services/tenant/index.ts

@@ -87,8 +87,6 @@ export async function install(
     throw new TenantInstalledAlreadyError();
   }
 
-  // TODO: (wyattjoh) perform any pending migrations.
-
   logger.info("installing tenant");
 
   // Create the Tenant.

src/core/server/services/users/delete.ts

@@ -1,22 +1,10 @@
 import { Collection, Db } from "mongodb";
 
-import { CommentAction } from "coral-server/models/action/comment";
-import { createCollection } from "coral-server/models/helpers";
 import { Story } from "coral-server/models/story";
-import { Tenant } from "coral-server/models/tenant";
-import { User } from "coral-server/models/user";
+import collections from "coral-server/services/mongodb/collections";
 
 const BATCH_SIZE = 500;
 
-// TODO: extract this out to a separate file so it can be re-used elsewhere
-const collections = {
-  users: createCollection<User>("users"),
-  comments: createCollection<Comment>("comments"),
-  stories: createCollection<Story>("stories"),
-  tenants: createCollection<Tenant>("tenants"),
-  commentActions: createCollection<CommentAction>("commentActions"),
-};
-
 async function executeBulkOperations<T>(
   collection: Collection<T>,
   operations: any[]

src/core/server/services/users/index.ts

@@ -820,8 +820,7 @@ export async function premod(
 
   // Check to see if the User is currently banned.
   const premodStatus = consolidateUserPremodStatus(targetUser.status.premod);
-  // FIXME: (wyattjoh) once migration has been performed, remove check
-  if (premodStatus && premodStatus.active) {
+  if (premodStatus.active) {
     throw new UserAlreadyPremoderated();
   }
 
@@ -845,8 +844,7 @@ export async function removePremod(
 
   // Check to see if the User is currently suspended.
   const premodStatus = consolidateUserPremodStatus(targetUser.status.premod);
-  // FIXME: (wyattjoh) once migration has been performed, remove check
-  if (!premodStatus || !premodStatus.active) {
+  if (!premodStatus.active) {
     // The user is not premodded currently, just return the user because we
     // don't have to do anything.
     return targetUser;