applied migration performance improvements

services/migration/helpers.js

@@ -1,3 +1,5 @@
+const debug = require('debug')('talk:services:migration');
+
 /**
  * processUpdates processes batches of updates on the given model.
  *
@@ -16,16 +18,37 @@ const processUpdates = async (model, updates) => {
   await bulk.execute();
 };
 
+const debugProcessStatistics = (count, totalCount) => {
+  if (totalCount > 0) {
+    debug(
+      `processed ${(count / totalCount * 100).toFixed(
+        2
+      )}% (${count}/${totalCount}) update`
+    );
+  } else {
+    debug(`processed ${count} updates`);
+  }
+};
+
 const transformSingleWithCursor = ({
   queryBatchSize,
   updateBatchSize,
-}) => async (cursor, process, Model) => {
+}) => async (query, process, Model) => {
+  debug('starting transform');
+
   // We'll manage the updates that we store inside this object.
   let updates = [];
 
-  // First we'll collect all the individual actions with specific group id's.
-  cursor = await cursor.batchSize(queryBatchSize);
+  // Count the elements in the transformation.
+  let totalCount = 0;
+  try {
+    totalCount = await query.count();
+  } catch (err) {}
 
+  // First we'll collect all the individual actions with specific group id's.
+  const cursor = await query.batchSize(queryBatchSize);
+
+  let count = 0;
   while (await cursor.hasNext()) {
     const result = await cursor.next();
 
@@ -37,6 +60,8 @@ const transformSingleWithCursor = ({
     if (updates.length > updateBatchSize) {
       // Process the updates.
       await processUpdates(Model, updates);
+      count += updates.length;
+      debugProcessStatistics(count, totalCount);
 
       // Clear the updates array.
       updates = [];
@@ -46,10 +71,14 @@ const transformSingleWithCursor = ({
   if (updates.length > 0) {
     // Process the updates.
     await processUpdates(Model, updates);
+    count += updates.length;
+    debugProcessStatistics(count, totalCount);
 
     // Clear the updates array.
     updates = [];
   }
+
+  debug('finished transform');
 };
 
 /**

services/migration/index.js

@@ -115,7 +115,7 @@ class MigrationService {
    */
   static async run(
     migrations,
-    { queryBatchSize = 100, updateBatchSize = 1000 } = {}
+    { queryBatchSize = 10000, updateBatchSize = 20000 } = {}
   ) {
     if (migrations.length === 0) {
       console.log('No migrations to run!');