[CORL-793] Scheduled Indexes (#2723)

* fix: modified digesting to use index

* fix: added index for deletion

* review: cleaning up

* review: fixed log copy

src/core/server/models/user/user.ts

@@ -420,6 +420,11 @@ export interface User extends TenantResource {
    */
   digests: Digest[];
 
+  /**
+   * hasDigests is true when there is digests to send.
+   */
+  hasDigests?: boolean;
+
   /**
    * status stores the user status information regarding moderation state.
    */
@@ -2270,6 +2275,9 @@ export async function insertUserNotificationDigests(
       $push: {
         digests: { $each: digests },
       },
+      $set: {
+        hasDigests: true,
+      },
     },
     {
       // False to return the updated document instead of the original
@@ -2308,9 +2316,9 @@ export async function pullUserNotificationDigests(
     {
       tenantID,
       "notifications.digestFrequency": frequency,
-      digests: { $ne: [] },
+      hasDigests: true,
     },
-    { $set: { digests: [] } },
+    { $set: { digests: [], hasDigests: false } },
     {
       // True to return the original document instead of the updated document.
       returnOriginal: true,

src/core/server/services/migrate/manager.ts

@@ -70,6 +70,15 @@ export default class Manager {
       const id = parseInt(matches[1], 10);
       const name = matches[2];
 
+      // Skip this migration if it was disabled.
+      if (m.default.disabled) {
+        logger.warn(
+          { migrationID: id, migrationName: name },
+          "skipping disabled migration"
+        );
+        continue;
+      }
+
       // Create the migration instance.
       const migration = new m.default({ id, name, i18n });
 

src/core/server/services/migrate/migration.ts

@@ -23,6 +23,11 @@ abstract class Migration {
   public readonly logger: Logger;
   public readonly i18n: I18n;
 
+  /**
+   * disabled when true will not run the migration.
+   */
+  public static disabled?: boolean;
+
   constructor({ id, name, i18n }: MigrationOptions) {
     this.id = id;
     this.name = name;

src/core/server/services/migrate/migration_sample.ts

@@ -7,6 +7,10 @@ import { Db } from "mongodb";
 import Migration from "coral-server/services/migrate/migration";
 
 export default class extends Migration {
+  // Remove the following line once the migration is ready, otherwise the
+  // migration will not be ran!
+  public static disabled = true;
+
   public async up(mongo: Db, tenantID: string) {
     throw new Error("migration not implemented");
   }

src/core/server/services/migrate/migrations/1569947670260_add_moderator_notes_to_user.ts

@@ -20,7 +20,7 @@ export default class extends Migration {
     this.log(tenantID).warn(
       {
         matchedCount: result.matchedCount,
-        modifiedCount: result.matchedCount,
+        modifiedCount: result.modifiedCount,
       },
       "added empty moderatorNotes array"
     );

src/core/server/services/migrate/migrations/1574287034612_notification_digest_index.ts

@@ -0,0 +1,46 @@
+import { Db } from "mongodb";
+
+import Migration from "coral-server/services/migrate/migration";
+import collections from "coral-server/services/mongodb/collections";
+
+import { createIndex } from "../indexing";
+
+export default class extends Migration {
+  public async up(mongo: Db, tenantID: string) {
+    const result = await collections.users(mongo).updateMany(
+      {
+        tenantID,
+        digests: {
+          $ne: [],
+        },
+      },
+      {
+        $set: {
+          hasDigests: true,
+        },
+      }
+    );
+    this.log(tenantID).warn(
+      {
+        matchedCount: result.matchedCount,
+        modifiedCount: result.modifiedCount,
+      },
+      "added hasDigests flag"
+    );
+  }
+
+  public async indexes(mongo: Db) {
+    await createIndex(
+      collections.users(mongo),
+      {
+        tenantID: 1,
+        "notifications.digestFrequency": 1,
+        hasDigests: 1,
+      },
+      {
+        partialFilterExpression: { hasDigests: { $eq: true } },
+        background: true,
+      }
+    );
+  }
+}

src/core/server/services/migrate/migrations/1574289134415_scheduled_deletion_date_index.ts

@@ -0,0 +1,22 @@
+import { Db } from "mongodb";
+
+import Migration from "coral-server/services/migrate/migration";
+import collections from "coral-server/services/mongodb/collections";
+
+import { createIndex } from "../indexing";
+
+export default class extends Migration {
+  public async indexes(mongo: Db) {
+    await createIndex(
+      collections.users(mongo),
+      {
+        tenantID: 1,
+        scheduledDeletionDate: 1,
+      },
+      {
+        partialFilterExpression: { scheduledDeletionDate: { $exists: true } },
+        background: true,
+      }
+    );
+  }
+}