migration rewrite and removed verifications

services/migration/helpers.js

@@ -0,0 +1,78 @@
+/**
+ * processUpdates processes batches of updates on the given model.
+ *
+ * @param {Object} model mongoose model that should perform the operations on
+ * @param {Array<Object>} updates array of updates to execute
+ */
+const processUpdates = async (model, updates) => {
+  // Create a new batch operation.
+  const bulk = model.collection.initializeUnorderedBulkOp();
+
+  for (const { query, update } of updates) {
+    bulk.find(query).updateOne(update);
+  }
+
+  // Execute the bulk update operation.
+  await bulk.execute();
+};
+
+const transformSingleWithCursor = ({
+  queryBatchSize,
+  updateBatchSize,
+}) => async (cursor, process, Model) => {
+  // We'll manage the updates that we store inside this object.
+  let updates = [];
+
+  // First we'll collect all the individual actions with specific group id's.
+  cursor = await cursor.batchSize(queryBatchSize);
+
+  while (await cursor.hasNext()) {
+    const result = await cursor.next();
+
+    const transformed = await process(result);
+    if (transformed) {
+      updates.push(transformed);
+    }
+
+    if (updates.length > updateBatchSize) {
+      // Process the updates.
+      await processUpdates(Model, updates);
+
+      // Clear the updates array.
+      updates = [];
+    }
+  }
+
+  if (updates.length > 0) {
+    // Process the updates.
+    await processUpdates(Model, updates);
+
+    // Clear the updates array.
+    updates = [];
+  }
+};
+
+/**
+ * processManyUpdates processes batches of updates on many models with the given
+ * model.
+ *
+ * @param {Object} model mongoose model that should perform the operations on
+ * @param {Array<Object>} updates array of updates to execute
+ */
+const processManyUpdates = async (model, updates) => {
+  // Create a new batch operation.
+  const bulk = model.collection.initializeUnorderedBulkOp();
+
+  for (const { query, update } of updates) {
+    bulk.find(query).update(update);
+  }
+
+  // Execute the bulk update operation.
+  await bulk.execute();
+};
+
+module.exports = ctx => ({
+  processManyUpdates,
+  processUpdates,
+  transformSingleWithCursor: transformSingleWithCursor(ctx),
+});

services/migration/index.js

@@ -1,12 +1,13 @@
-const MigrationModel = require('../models/migration');
+const MigrationModel = require('../../models/migration');
 const fs = require('fs');
 const ms = require('ms');
 const path = require('path');
 const Joi = require('joi');
 const debug = require('debug')('talk:services:migration');
 const sc = require('snake-case');
+const helpers = require('./helpers');
 const { stripIndent } = require('common-tags');
-const { talk: { migration: { minVersion } } } = require('../package.json');
+const { talk: { migration: { minVersion } } } = require('../../package.json');
 
 const migrationTemplate = stripIndent`
   module.exports = {
@@ -46,7 +47,7 @@ class MigrationService {
   static async listPending() {
     // Get all the migration files.
     let migrationFiles = fs.readdirSync(
-      path.join(__dirname, '..', 'migrations')
+      path.join(__dirname, '..', '..', 'migrations')
     );
 
     // Ensure that all migrations follow this format.
@@ -78,7 +79,7 @@ class MigrationService {
         }
 
         // Read the migration from the filesystem.
-        let migration = require(`../migrations/${filename}`);
+        let migration = require(`../../migrations/${filename}`);
         Joi.assert(
           migration,
           migrationSchema,
@@ -121,11 +122,14 @@ class MigrationService {
       return;
     }
 
+    // Create the context helpers.
+    const ctx = helpers({ queryBatchSize, updateBatchSize });
+
     for (let { filename, version, migration } of migrations) {
       try {
         const startTime = new Date();
         console.log(`Starting migration ${filename}`);
-        await migration.up({ queryBatchSize, updateBatchSize });
+        await migration.up(ctx);
         const endTime = new Date();
         const totalTime = endTime.getTime() - startTime.getTime();
         console.log(`Finished migration ${filename} in ${ms(totalTime)}`);