diff --git a/bin/cli b/bin/cli index b61c529c5..314f33dd4 100755 --- a/bin/cli +++ b/bin/cli @@ -18,7 +18,6 @@ program .command('token', 'work with the access tokens') .command('users', 'work with the application auth') .command('migration', 'provides utilities for migrating the database') - .command('verify', 'provides utilities for performing data verification') .command( 'plugins', 'provides utilities for interacting with the plugin system' diff --git a/bin/cli-migration b/bin/cli-migration index ca93360ad..63e16f97f 100755 --- a/bin/cli-migration +++ b/bin/cli-migration @@ -5,6 +5,7 @@ */ const util = require('./util'); +const _ = require('lodash'); const program = require('commander'); const inquirer = require('inquirer'); const mongoose = require('../services/mongoose'); @@ -25,46 +26,60 @@ async function createMigration(name) { } } -async function runMigrations() { +async function runMigrations(options) { + const { yes, queryBatchSize, updateBatchSize } = options; try { - let { backedUp } = await inquirer.prompt([ - { - type: 'confirm', - name: 'backedUp', - message: 'Did you perform a database backup', - default: false, - }, - ]); + if (!yes) { + const { backedUp } = await inquirer.prompt([ + { + type: 'confirm', + name: 'backedUp', + message: 'Did you perform a database backup', + default: false, + }, + ]); - if (!backedUp) { - throw new Error( - 'Please backup your databases prior to migrations occuring' - ); + if (!backedUp) { + throw new Error( + 'Please backup your databases prior to migrations occuring' + ); + } } // Get the migrations to run. - let migrations = await MigrationService.listPending(); + const migrations = await MigrationService.listPending(); console.log('Now going to run the following migrations:\n'); - for (let { filename } of migrations) { + for (const { filename } of migrations) { console.log(`\tmigrations/${filename}`); } - let { confirm } = await inquirer.prompt([ - { - type: 'confirm', - name: 'confirm', - message: 'Proceed with migrations', - default: false, - }, - ]); + if (!yes) { + const { confirm } = await inquirer.prompt([ + { + type: 'confirm', + name: 'confirm', + message: 'Proceed with migrations', + default: false, + }, + ]); - if (confirm) { - // Run the migrations. - await MigrationService.run(migrations); + if (confirm) { + // Run the migrations. + await MigrationService.run(migrations, { + queryBatchSize, + updateBatchSize, + }); + } else { + console.warn('Skipping migrations'); + } } else { - console.warn('Skipping migrations'); + // Run the migrations. + await MigrationService.run(migrations, { + queryBatchSize, + updateBatchSize, + }); } util.shutdown(); @@ -83,8 +98,25 @@ program .description('creates a new migration') .action(createMigration); +// Bypasses issue that defaults + coercion doesn't work well together. +// Ref: https://github.com/tj/commander.js/issues/400#issuecomment-310860869 +const parse10 = _.ary(_.partialRight(parseInt, 10), 1); + program .command('run') + .option( + '-q, --query-batch-size ', + 'change the size of queried documents that are batched at a time', + parse10, + 10000 + ) + .option( + '-u, --update-batch-size ', + 'change the size of documents that are batched before the update is sent', + parse10, + 20000 + ) + .option('-y, --yes', 'will answer yes to all questions') .description('runs all pending migrations') .action(runMigrations); diff --git a/bin/cli-setup b/bin/cli-setup index b0f5e2efa..282a8a2d7 100755 --- a/bin/cli-setup +++ b/bin/cli-setup @@ -13,6 +13,7 @@ const MODERATION_OPTIONS = require('../models/enum/moderation_options'); const SettingsService = require('../services/settings'); const SetupService = require('../services/setup'); const UsersService = require('../services/users'); +const MigrationService = require('../services/migration'); const errors = require('../errors'); // Register the shutdown criteria. @@ -51,6 +52,12 @@ const performSetup = async () => { if (program.defaults) { await SettingsService.init(); + // Get the migrations to run. + let migrations = await MigrationService.listPending(); + + // Perform all migrations. + await MigrationService.run(migrations); + console.log('Settings created.'); console.log('\nTalk is now installed!'); @@ -194,7 +201,7 @@ const performSetup = async () => { ); }; -// Start tthe setup process. +// Start the setup process. performSetup() .then(() => { util.shutdown(); diff --git a/bin/cli-verify b/bin/cli-verify deleted file mode 100755 index e32c7d169..000000000 --- a/bin/cli-verify +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env node - -/** - * Module dependencies. - */ - -const util = require('./util'); -const program = require('commander'); -const mongoose = require('../services/mongoose'); -const databaseVerifications = require('./verifications/database'); - -// Register the shutdown criteria. -util.onshutdown([() => mongoose.disconnect()]); - -async function database({ fix = false, limit = Infinity, batch = 1000 }) { - try { - for (const verification of databaseVerifications) { - await verification({ fix, limit, batch }); - } - } catch (err) { - console.error( - `Failed to process all the ${databaseVerifications.length} verifications`, - err - ); - util.shutdown(1); - return; - } - - util.shutdown(); -} - -//============================================================================== -// Setting up the program command line arguments. -//============================================================================== - -program - .command('db') - .description('verifies the database integrity') - .option('-f, --fix', 'fix the problems found with database inconsistencies') - .option( - '-l, --limit [size]', - 'limit the amount of documents to process in a single pass, this will ensure only a maximum number of batch operations are issued [default: inf]', - parseInt - ) - .option( - '-b, --batch [size]', - 'batch size to process verifications and repairs of documents [default: 1000]', - parseInt - ) - .action(database); - -program.parse(process.argv); - -// If there is no command listed, output help. -if (!process.argv.slice(2).length) { - program.outputHelp(); - util.shutdown(); -} diff --git a/docs/_docs/06-01-migrating-4.md b/docs/_docs/06-01-migrating-4.md index b66921cf0..c8af6844d 100644 --- a/docs/_docs/06-01-migrating-4.md +++ b/docs/_docs/06-01-migrating-4.md @@ -40,28 +40,6 @@ documents rather than performing a nice table alter. If the process crashes during the migration, simply re-run it. The migration operations are designed to act atomically, and be idempotent to documents already updated. -## Database Verifications - -In `v3.*`, we introduced the concept of "verifying the database". Some of our -operations update cached values that live along side the original document to -improve performance. Running the cli command for verifying the database's cache -ensures that all the cached values are up to date. - -Running the following will start the database verification process: - -```bash -./bin/cli verify db --fix -``` -You can notice the `--fix` option, without it, the tool should instead perform -a dry run of the operations it intends to perform. -{: .code-aside} - -This process, like the migration process, should take some time to complete on -large databases. - -Once you have updated your databases, that's all you have to do! Talk should now -function even better and faster with all the new features we poured into v4.0.0! - ## Template Change In `v4.0.0`, we introduced extensive support for compressing our javascript diff --git a/migrations/1496771633_tags.js b/migrations/1496771633_tags.js index b21ff671e..6893a5c47 100644 --- a/migrations/1496771633_tags.js +++ b/migrations/1496771633_tags.js @@ -1,65 +1,38 @@ const CommentModel = require('../models/comment'); -module.exports = { - async up() { - // Find all comments that have tags. - let comments = await CommentModel.aggregate([ - { - $match: { - tags: { - $exists: true, - $ne: [], - }, - }, - }, - { - $project: { - id: true, - tags: true, - }, - }, - ]); +// OLD +// +// [ +// { +// name: 'OFF_TOPIC', +// assigned_by: '', +// created_at: new Date() +// } +// ] - // If no comments were found, nothing needs to be done! - if (comments.length <= 0) { - return; - } - - const updates = []; - - // Loop over the comments retrieved, updating the tag structure. - for (let { id, tags } of comments) { - // OLD - // - // [ - // { - // name: 'OFF_TOPIC', - // assigned_by: '', - // created_at: new Date() - // } - // ] - - // NEW - // - // [ - // { - // tag: { - // name: 'OFF_TOPIC', - // permissions: { - // public: true, - // self: false, - // roles: [] - // }, - // models: ['COMMENTS'], - // created_at: new Date() - // }, - // assigned_by: '', - // created_at: new Date() - // } - // ] - - // Remap the tag structure. - tags = tags.map(({ name, assigned_by, created_at }) => ({ +// NEW +// +// [ +// { +// tag: { +// name: 'OFF_TOPIC', +// permissions: { +// public: true, +// self: false, +// roles: [] +// }, +// models: ['COMMENTS'], +// created_at: new Date() +// }, +// assigned_by: '', +// created_at: new Date() +// } +// ] +const transformTags = ({ id, tags }) => ({ + query: { id }, + update: { + $set: { + tags: tags.map(({ name, assigned_by, created_at }) => ({ tag: { name, permissions: { @@ -72,22 +45,34 @@ module.exports = { }, assigned_by, created_at, - })); + })), + }, + }, +}); - updates.push({ query: { id }, update: { $set: { tags } } }); - } +module.exports = { + async up({ transformSingleWithCursor }) { + // Find all comments that have tags. + const cursor = CommentModel.collection.aggregate( + [ + { + $match: { + tags: { + $exists: true, + $ne: [], + }, + }, + }, + { + $project: { + id: true, + tags: true, + }, + }, + ], + { allowDiskUse: true } + ); - if (updates.length > 0) { - // Create a new batch operation. - let batch = CommentModel.collection.initializeUnorderedBulkOp(); - - for (const { query, update } of updates) { - // Execute the batch operation. - batch.find(query).updateOne(update); - } - - // Execute the batch update operation. - await batch.execute(); - } + await transformSingleWithCursor(cursor, transformTags, CommentModel); }, }; diff --git a/migrations/1507310316_flags.js b/migrations/1507310316_flags.js index d29a2c25b..1ff3b4248 100644 --- a/migrations/1507310316_flags.js +++ b/migrations/1507310316_flags.js @@ -12,7 +12,7 @@ const mapping = { }; module.exports = { - async up() { + async up({ processManyUpdates }) { const updates = []; for (const item_type in mapping) { const mappings = mapping[item_type]; @@ -44,15 +44,7 @@ module.exports = { } if (updates.length > 0) { - // Setup the batch operation. - const batch = ActionModel.collection.initializeUnorderedBulkOp(); - - for (const { query, update } of updates) { - batch.find(query).update(update); - } - - // Execute the batch update operation. - await batch.execute(); + await processManyUpdates(ActionModel, updates); } }, }; diff --git a/migrations/1510174676_user_status.js b/migrations/1510174676_user_status.js index b3d7c0984..80271557b 100644 --- a/migrations/1510174676_user_status.js +++ b/migrations/1510174676_user_status.js @@ -1,60 +1,132 @@ const UserModel = require('../models/user'); const merge = require('lodash/merge'); -const getUserBatch = async () => { - let query = { - status: { - $in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'], +const transformUser = user => { + const created_at = Date.now(); + + const { id, status, canEditName, suspension, disabled } = user; + + let update = { + $unset: { + canEditName: '', + suspension: '', + disabled: '', + }, + $set: { + status: { + // The username status is specific to each case. + username: { + history: [], + }, + + // The user is not banned by default. + banned: { + status: false, + history: [], + }, + + // The user is not suspended by default. + suspension: { + until: null, + history: [], + }, + }, + updated_at: created_at, }, }; - // Find all the users that need migrating. - return UserModel.collection.find(query); -}; - -module.exports = { - async up() { - const created_at = Date.now(); - - // Get the first batch of users. - let cursor = await getUserBatch(); - - const updates = []; - while (await cursor.hasNext()) { - const user = await cursor.next(); - - const { id, status, canEditName, suspension, disabled } = user; - - let update = { - $unset: { - canEditName: '', - suspension: '', - disabled: '', + if (disabled) { + update = merge(update, { + $set: { + status: { + banned: { + status: true, + history: [ + { + status: true, + created_at, + }, + ], + }, }, - $set: { - status: { - // The username status is specific to each case. - username: { - history: [], - }, + }, + }); + } - // The user is not banned by default. - banned: { - status: false, - history: [], - }, + // If the user has an "until" property of their suspension, then we need + // to reflect that in the new status object. + if (suspension && suspension.until !== null) { + update = merge(update, { + $set: { + status: { + suspension: { + until: suspension.until, + history: [ + { + until: suspension.until, + created_at, + }, + ], + }, + }, + }, + }); + } - // The user is not suspended by default. - suspension: { - until: null, - history: [], + switch (status) { + case 'ACTIVE': + if (canEditName) { + update = merge(update, { + $set: { + status: { + username: { + status: 'UNSET', + history: [ + { + status: 'UNSET', + created_at, + }, + ], + }, }, }, - updated_at: created_at, - }, - }; - - if (disabled) { + }); + } else { + update = merge(update, { + $set: { + status: { + username: { + status: 'SET', + history: [ + { + status: 'SET', + created_at, + }, + ], + }, + }, + }, + }); + } + break; + case 'BANNED': + if (canEditName) { + update = merge(update, { + $set: { + status: { + username: { + status: 'REJECTED', + history: [ + { + status: 'REJECTED', + created_at, + }, + ], + }, + }, + }, + }); + } else { update = merge(update, { $set: { status: { @@ -67,22 +139,11 @@ module.exports = { }, ], }, - }, - }, - }); - } - - // If the user has an "until" property of their suspension, then we need - // to reflect that in the new status object. - if (suspension && suspension.until !== null) { - update = merge(update, { - $set: { - status: { - suspension: { - until: suspension.until, + username: { + status: 'SET', history: [ { - until: suspension.until, + status: 'SET', created_at, }, ], @@ -91,138 +152,57 @@ module.exports = { }, }); } - - switch (status) { - case 'ACTIVE': - if (canEditName) { - update = merge(update, { - $set: { - status: { - username: { - status: 'UNSET', - history: [ - { - status: 'UNSET', - created_at, - }, - ], - }, - }, - }, - }); - } else { - update = merge(update, { - $set: { - status: { - username: { - status: 'SET', - history: [ - { - status: 'SET', - created_at, - }, - ], - }, - }, - }, - }); - } - break; - case 'BANNED': - if (canEditName) { - update = merge(update, { - $set: { - status: { - username: { - status: 'REJECTED', - history: [ - { - status: 'REJECTED', - created_at, - }, - ], - }, - }, - }, - }); - } else { - update = merge(update, { - $set: { - status: { - banned: { - status: true, - history: [ - { - status: true, - created_at, - }, - ], - }, - username: { - status: 'SET', - history: [ - { - status: 'SET', - created_at, - }, - ], - }, - }, - }, - }); - } - break; - case 'PENDING': - update = merge(update, { - $set: { - status: { - username: { + break; + case 'PENDING': + update = merge(update, { + $set: { + status: { + username: { + status: 'CHANGED', + history: [ + { status: 'CHANGED', - history: [ - { - status: 'CHANGED', - created_at, - }, - ], + created_at, }, - }, + ], }, - }); - break; - case 'APPROVED': - update = merge(update, { - $set: { - status: { - username: { + }, + }, + }); + break; + case 'APPROVED': + update = merge(update, { + $set: { + status: { + username: { + status: 'APPROVED', + history: [ + { status: 'APPROVED', - history: [ - { - status: 'APPROVED', - created_at, - }, - ], + created_at, }, - }, + ], }, - }); - break; - default: - throw new Error(`${status} is an invalid status`); - } + }, + }, + }); + break; + default: + throw new Error(`${status} is an invalid status`); + } - updates.push({ query: { id }, update }); - } + return { query: { id }, update }; +}; - if (updates.length > 0) { - // Create a new batch operation. - let bulk = UserModel.collection.initializeUnorderedBulkOp(); +module.exports = { + async up({ transformSingleWithCursor }) { + // Get the first batch of users. + const cursor = UserModel.collection.find({ + status: { + $in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'], + }, + }); - for (const { query, update } of updates) { - bulk.find(query).updateOne(update); - } - - // Execute the bulk update operation. - await bulk.execute(); - } + await transformSingleWithCursor(cursor, transformUser, UserModel); }, }; diff --git a/migrations/1511801783_user_roles.js b/migrations/1511801783_user_roles.js index 08a0a1abc..961a696e2 100644 --- a/migrations/1511801783_user_roles.js +++ b/migrations/1511801783_user_roles.js @@ -13,18 +13,16 @@ const findNewRole = roles => { }; module.exports = { - async up() { - const cursor = await UserModel.collection.find({ + async up({ transformSingleWithCursor }) { + const cursor = UserModel.collection.find({ roles: { $exists: true, }, }); - const updates = []; - while (await cursor.hasNext()) { - const user = await cursor.next(); - - updates.push({ + await transformSingleWithCursor( + cursor, + user => ({ query: { id: user.id, }, @@ -38,19 +36,8 @@ module.exports = { roles: '', }, }, - }); - } - - if (updates.length > 0) { - // Create a new batch operation. - const bulk = UserModel.collection.initializeUnorderedBulkOp(); - - for (const { query, update } of updates) { - bulk.find(query).updateOne(update); - } - - // Execute the bulk update operation. - await bulk.execute(); - } + }), + UserModel + ); }, }; diff --git a/migrations/1516920154_action_counts.js b/migrations/1516920154_action_counts.js new file mode 100644 index 000000000..7558b5eb0 --- /dev/null +++ b/migrations/1516920154_action_counts.js @@ -0,0 +1,124 @@ +const ActionModel = require('../models/action'); +const UserModel = require('../models/user'); +const CommentModel = require('../models/comment'); + +module.exports = { + async up({ transformSingleWithCursor }) { + const models = [ + { Model: CommentModel, item_type: 'COMMENTS' }, + { Model: UserModel, item_type: 'USERS' }, + ]; + for (const { Model, item_type } of models) { + let cursor = ActionModel.collection.aggregate( + [ + { + $match: { + group_id: { $ne: null }, + item_type, + }, + }, + { + $group: { + // group unique documents by these properties, we are leveraging the + // fact that each uuid is completely unique. + _id: { + item_id: '$item_id', + action_type: '$action_type', + group_id: '$group_id', + }, + + // and sum up all actions matching the above grouping criteria + count: { + $sum: 1, + }, + }, + }, + { + $project: { + // suppress the _id field + _id: false, + + // map the fields from the _id grouping down a level + item_id: '$_id.item_id', + action_type: { $toLower: '$_id.action_type' }, + group_id: { $toLower: '$_id.group_id' }, + + // map the field directly + count: '$count', + }, + }, + ], + { allowDiskUse: true } + ); + + // Transform those documents. + await transformSingleWithCursor( + cursor, + ({ item_id, action_type, group_id, count }) => ({ + query: { id: item_id }, + update: { + $set: { + [`action_counts.${action_type}_${group_id}`]: count, + }, + }, + }), + Model + ); + + // Secondly, we'll collect the group group id's (all the actions for a + // specific action type) to update counts of. + cursor = ActionModel.collection.aggregate( + [ + { + $match: { + item_type, + }, + }, + { + $group: { + // group unique documents by these properties, we are leveraging the + // fact that each uuid is completely unique. + _id: { + item_id: '$item_id', + action_type: '$action_type', + }, + + // and sum up all actions matching the above grouping criteria + count: { + $sum: 1, + }, + }, + }, + { + $project: { + // suppress the _id field + _id: false, + + // map the fields from the _id grouping down a level + item_id: '$_id.item_id', + action_type: { $toLower: '$_id.action_type' }, + + // map the field directly + count: '$count', + }, + }, + ], + { allowDiskUse: true } + ); + + // Transform those documents. + await transformSingleWithCursor( + cursor, + ({ item_id, action_type, count }) => ({ + query: { id: item_id }, + update: { + $set: { + [`action_counts.${action_type}`]: count, + }, + }, + }), + Model + ); + } + }, +}; diff --git a/migrations/1516920160_reply_counts.js b/migrations/1516920160_reply_counts.js new file mode 100644 index 000000000..fc81bd912 --- /dev/null +++ b/migrations/1516920160_reply_counts.js @@ -0,0 +1,33 @@ +const CommentModel = require('../models/comment'); + +const transformComments = ({ _id: parent_id, reply_count }) => ({ + query: { id: parent_id, reply_count: { $ne: reply_count } }, + update: { $set: { reply_count } }, +}); + +module.exports = { + async up({ transformSingleWithCursor }) { + const cursor = CommentModel.collection.aggregate( + [ + { + $match: { + parent_id: { $ne: null }, + status: { $in: ['NONE', 'ACCEPTED'] }, + }, + }, + { + $group: { + _id: '$parent_id', + reply_count: { + $sum: 1, + }, + }, + }, + ], + { allowDiskUse: true } + ); + + // Transform those documents. + await transformSingleWithCursor(cursor, transformComments, CommentModel); + }, +}; diff --git a/package.json b/package.json index 17af19867..b3afdc9b6 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "talk": { "migration": { - "minVersion": 1511801783 + "minVersion": 1516920160 } }, "repository": { @@ -222,6 +222,10 @@ }, "pre-commit": { "silent": false, - "run": ["lint", "test:client", "test:server"] + "run": [ + "lint", + "test:client", + "test:server" + ] } } diff --git a/services/migration/helpers.js b/services/migration/helpers.js new file mode 100644 index 000000000..5eb33aadf --- /dev/null +++ b/services/migration/helpers.js @@ -0,0 +1,107 @@ +const debug = require('debug')('talk:services:migration'); + +/** + * processUpdates processes batches of updates on the given model. + * + * @param {Object} model mongoose model that should perform the operations on + * @param {Array} updates array of updates to execute + */ +const processUpdates = async (model, updates) => { + // Create a new batch operation. + const bulk = model.collection.initializeUnorderedBulkOp(); + + for (const { query, update } of updates) { + bulk.find(query).updateOne(update); + } + + // Execute the bulk update operation. + await bulk.execute(); +}; + +const debugProcessStatistics = (count, totalCount) => { + if (totalCount > 0) { + debug( + `processed ${(count / totalCount * 100).toFixed( + 2 + )}% (${count}/${totalCount}) updates` + ); + } else { + debug(`processed ${count} updates`); + } +}; + +const transformSingleWithCursor = ({ + queryBatchSize, + updateBatchSize, +}) => async (query, process, Model) => { + debug('starting transform'); + + // We'll manage the updates that we store inside this object. + let updates = []; + + // Count the elements in the transformation. + let totalCount = 0; + try { + totalCount = await query.count(); + } catch (err) {} + + // First we'll collect all the individual actions with specific group id's. + const cursor = await query.batchSize(queryBatchSize); + + let count = 0; + while (await cursor.hasNext()) { + const result = await cursor.next(); + + const transformed = await process(result); + if (transformed) { + updates.push(transformed); + } + + if (updates.length > updateBatchSize) { + // Process the updates. + await processUpdates(Model, updates); + count += updates.length; + debugProcessStatistics(count, totalCount); + + // Clear the updates array. + updates = []; + } + } + + if (updates.length > 0) { + // Process the updates. + await processUpdates(Model, updates); + count += updates.length; + debugProcessStatistics(count, totalCount); + + // Clear the updates array. + updates = []; + } + + debug('finished transform'); +}; + +/** + * processManyUpdates processes batches of updates on many models with the given + * model. + * + * @param {Object} model mongoose model that should perform the operations on + * @param {Array} updates array of updates to execute + */ +const processManyUpdates = async (model, updates) => { + // Create a new batch operation. + const bulk = model.collection.initializeUnorderedBulkOp(); + + for (const { query, update } of updates) { + bulk.find(query).update(update); + } + + // Execute the bulk update operation. + await bulk.execute(); +}; + +module.exports = ctx => ({ + processManyUpdates, + processUpdates, + transformSingleWithCursor: transformSingleWithCursor(ctx), +}); diff --git a/services/migration.js b/services/migration/index.js similarity index 82% rename from services/migration.js rename to services/migration/index.js index f8f09d469..e0da3cddf 100644 --- a/services/migration.js +++ b/services/migration/index.js @@ -1,17 +1,20 @@ -const MigrationModel = require('../models/migration'); +const MigrationModel = require('../../models/migration'); const fs = require('fs'); +const ms = require('ms'); const path = require('path'); const Joi = require('joi'); const debug = require('debug')('talk:services:migration'); const sc = require('snake-case'); -const { talk: { migration: { minVersion } } } = require('../package.json'); +const helpers = require('./helpers'); +const { stripIndent } = require('common-tags'); +const { talk: { migration: { minVersion } } } = require('../../package.json'); -const migrationTemplate = `module.exports = { - async up() { - - } -}; +const migrationTemplate = stripIndent` + module.exports = { + async up({ queryBatchSize, updateBatchSize }) { + } + }; `; class MigrationService { @@ -44,7 +47,7 @@ class MigrationService { static async listPending() { // Get all the migration files. let migrationFiles = fs.readdirSync( - path.join(__dirname, '..', 'migrations') + path.join(__dirname, '..', '..', 'migrations') ); // Ensure that all migrations follow this format. @@ -61,6 +64,7 @@ class MigrationService { // Parse the migrations from the file listing. let migrations = migrationFiles + .filter(filename => versionRe.test(filename)) .map(filename => { // Parse the version from the filename. let matches = filename.match(versionRe); @@ -75,7 +79,7 @@ class MigrationService { } // Read the migration from the filesystem. - let migration = require(`../migrations/${filename}`); + let migration = require(`../../migrations/${filename}`); Joi.assert( migration, migrationSchema, @@ -109,17 +113,26 @@ class MigrationService { * * @param {Array} migrations a list of migrations returned by `listPending` */ - static async run(migrations) { + static async run( + migrations, + { queryBatchSize = 10000, updateBatchSize = 20000 } = {} + ) { if (migrations.length === 0) { console.log('No migrations to run!'); return; } + // Create the context helpers. + const ctx = helpers({ queryBatchSize, updateBatchSize }); + for (let { filename, version, migration } of migrations) { try { + const startTime = new Date(); console.log(`Starting migration ${filename}`); - await migration.up(); - console.log(`Finished migration ${filename}`); + await migration.up(ctx); + const endTime = new Date(); + const totalTime = endTime.getTime() - startTime.getTime(); + console.log(`Finished migration ${filename} in ${ms(totalTime)}`); } catch (e) { console.error(`Migration ${filename} failed`); throw e; diff --git a/test/server/migrations/1510174676_user_status.js b/test/server/migrations/1510174676_user_status.js index 8e1ceca5d..f9af655d8 100644 --- a/test/server/migrations/1510174676_user_status.js +++ b/test/server/migrations/1510174676_user_status.js @@ -1,10 +1,14 @@ const migration = require('../../../migrations/1510174676_user_status'); const UserModel = require('../../../models/user'); +const helpers = require('../../../services/migration/helpers'); const chai = require('chai'); chai.use(require('chai-datetime')); const { expect } = chai; +const performMigration = () => + migration.up(helpers({ queryBatchSize: 100, updateBatchSize: 100 })); + describe('migration.1510174676_user_status', () => { describe('active user', () => { beforeEach(async () => { @@ -24,7 +28,7 @@ describe('migration.1510174676_user_status', () => { expect(user).to.have.property('canEditName', false); // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); @@ -54,7 +58,7 @@ describe('migration.1510174676_user_status', () => { expect(user).to.have.property('canEditName', true); // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); @@ -85,7 +89,7 @@ describe('migration.1510174676_user_status', () => { expect(user.canEditName).to.equal(true); // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); @@ -117,7 +121,7 @@ describe('migration.1510174676_user_status', () => { expect(user.canEditName).to.equal(false); // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); @@ -153,7 +157,7 @@ describe('migration.1510174676_user_status', () => { const until = user.suspension.until; // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); @@ -187,7 +191,7 @@ describe('migration.1510174676_user_status', () => { expect(user.status).to.equal('BANNED'); // Perform the migration. - await migration.up(); + await performMigration(); user = await UserModel.collection.findOne({ id: '123' }); diff --git a/yarn.lock b/yarn.lock index 38f5963ec..d13397fee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -55,8 +55,8 @@ to-fast-properties "^2.0.0" "@coralproject/eslint-config-talk@^0.1.0": - version "0.1.0" - resolved "https://registry.yarnpkg.com/@coralproject/eslint-config-talk/-/eslint-config-talk-0.1.0.tgz#3ddc5f6fb4362a1cd05a5fea56cdb3095afc8cc3" + version "0.1.1" + resolved "https://registry.yarnpkg.com/@coralproject/eslint-config-talk/-/eslint-config-talk-0.1.1.tgz#71991b4937a3ffe657128d7f1170da4b5fb75c9e" dependencies: babel-eslint "^8.0.1" eslint-config-prettier "^2.9.0" @@ -2507,7 +2507,13 @@ dns-prefetch-control@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz#60ddb457774e178f1f9415f0cabb0e85b0b300b2" -doctrine@^2.0.0, doctrine@^2.0.2: +doctrine@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d" + dependencies: + esutils "^2.0.2" + +doctrine@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.2.tgz#68f96ce8efc56cc42651f1faadb4f175273b0075" dependencies: @@ -2746,7 +2752,7 @@ error-ex@^1.2.0, error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.4.3, es-abstract@^1.7.0: +es-abstract@^1.4.3: version "1.9.0" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.9.0.tgz#690829a07cae36b222e7fd9b75c0d0573eb25227" dependencies: @@ -2756,7 +2762,7 @@ es-abstract@^1.4.3, es-abstract@^1.7.0: is-callable "^1.1.3" is-regex "^1.0.4" -es-abstract@^1.6.1: +es-abstract@^1.6.1, es-abstract@^1.7.0: version "1.10.0" resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.10.0.tgz#1ecb36c197842a00d8ee4c2dfd8646bb97d60864" dependencies: @@ -2873,8 +2879,8 @@ eslint-config-prettier@^2.9.0: get-stdin "^5.0.1" eslint-plugin-jest@^21.6.1: - version "21.6.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-21.6.1.tgz#adca015bbdb8d23b210438ff9e1cee1dd9ec35df" + version "21.7.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-21.7.0.tgz#651f1c6ce999af3ac59ab8bf8a376d742fd0fc23" eslint-plugin-mocha@^4.11.0: version "4.11.0" @@ -2883,8 +2889,8 @@ eslint-plugin-mocha@^4.11.0: ramda "^0.24.1" eslint-plugin-prettier@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-2.4.0.tgz#85cab0775c6d5e3344ef01e78d960f166fb93aae" + version "2.5.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-2.5.0.tgz#39a91dd7528eaf19cd42c0ee3f2c1f684606a05f" dependencies: fast-diff "^1.1.1" jest-docblock "^21.0.0"