mirror of
https://github.com/wassname/talk.git
synced 2026-07-08 01:39:30 +08:00
Merge branch 'master' into cleanup
This commit is contained in:
@@ -18,7 +18,6 @@ program
|
||||
.command('token', 'work with the access tokens')
|
||||
.command('users', 'work with the application auth')
|
||||
.command('migration', 'provides utilities for migrating the database')
|
||||
.command('verify', 'provides utilities for performing data verification')
|
||||
.command(
|
||||
'plugins',
|
||||
'provides utilities for interacting with the plugin system'
|
||||
|
||||
+59
-27
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
const util = require('./util');
|
||||
const _ = require('lodash');
|
||||
const program = require('commander');
|
||||
const inquirer = require('inquirer');
|
||||
const mongoose = require('../services/mongoose');
|
||||
@@ -25,46 +26,60 @@ async function createMigration(name) {
|
||||
}
|
||||
}
|
||||
|
||||
async function runMigrations() {
|
||||
async function runMigrations(options) {
|
||||
const { yes, queryBatchSize, updateBatchSize } = options;
|
||||
try {
|
||||
let { backedUp } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'backedUp',
|
||||
message: 'Did you perform a database backup',
|
||||
default: false,
|
||||
},
|
||||
]);
|
||||
if (!yes) {
|
||||
const { backedUp } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'backedUp',
|
||||
message: 'Did you perform a database backup',
|
||||
default: false,
|
||||
},
|
||||
]);
|
||||
|
||||
if (!backedUp) {
|
||||
throw new Error(
|
||||
'Please backup your databases prior to migrations occuring'
|
||||
);
|
||||
if (!backedUp) {
|
||||
throw new Error(
|
||||
'Please backup your databases prior to migrations occuring'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Get the migrations to run.
|
||||
let migrations = await MigrationService.listPending();
|
||||
const migrations = await MigrationService.listPending();
|
||||
|
||||
console.log('Now going to run the following migrations:\n');
|
||||
|
||||
for (let { filename } of migrations) {
|
||||
for (const { filename } of migrations) {
|
||||
console.log(`\tmigrations/${filename}`);
|
||||
}
|
||||
|
||||
let { confirm } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'confirm',
|
||||
message: 'Proceed with migrations',
|
||||
default: false,
|
||||
},
|
||||
]);
|
||||
if (!yes) {
|
||||
const { confirm } = await inquirer.prompt([
|
||||
{
|
||||
type: 'confirm',
|
||||
name: 'confirm',
|
||||
message: 'Proceed with migrations',
|
||||
default: false,
|
||||
},
|
||||
]);
|
||||
|
||||
if (confirm) {
|
||||
// Run the migrations.
|
||||
await MigrationService.run(migrations);
|
||||
if (confirm) {
|
||||
// Run the migrations.
|
||||
await MigrationService.run(migrations, {
|
||||
queryBatchSize,
|
||||
updateBatchSize,
|
||||
});
|
||||
} else {
|
||||
console.warn('Skipping migrations');
|
||||
}
|
||||
} else {
|
||||
console.warn('Skipping migrations');
|
||||
// Run the migrations.
|
||||
await MigrationService.run(migrations, {
|
||||
queryBatchSize,
|
||||
updateBatchSize,
|
||||
});
|
||||
}
|
||||
|
||||
util.shutdown();
|
||||
@@ -83,8 +98,25 @@ program
|
||||
.description('creates a new migration')
|
||||
.action(createMigration);
|
||||
|
||||
// Bypasses issue that defaults + coercion doesn't work well together.
|
||||
// Ref: https://github.com/tj/commander.js/issues/400#issuecomment-310860869
|
||||
const parse10 = _.ary(_.partialRight(parseInt, 10), 1);
|
||||
|
||||
program
|
||||
.command('run')
|
||||
.option(
|
||||
'-q, --query-batch-size <n>',
|
||||
'change the size of queried documents that are batched at a time',
|
||||
parse10,
|
||||
10000
|
||||
)
|
||||
.option(
|
||||
'-u, --update-batch-size <n>',
|
||||
'change the size of documents that are batched before the update is sent',
|
||||
parse10,
|
||||
20000
|
||||
)
|
||||
.option('-y, --yes', 'will answer yes to all questions')
|
||||
.description('runs all pending migrations')
|
||||
.action(runMigrations);
|
||||
|
||||
|
||||
+8
-1
@@ -13,6 +13,7 @@ const MODERATION_OPTIONS = require('../models/enum/moderation_options');
|
||||
const SettingsService = require('../services/settings');
|
||||
const SetupService = require('../services/setup');
|
||||
const UsersService = require('../services/users');
|
||||
const MigrationService = require('../services/migration');
|
||||
const errors = require('../errors');
|
||||
|
||||
// Register the shutdown criteria.
|
||||
@@ -51,6 +52,12 @@ const performSetup = async () => {
|
||||
if (program.defaults) {
|
||||
await SettingsService.init();
|
||||
|
||||
// Get the migrations to run.
|
||||
let migrations = await MigrationService.listPending();
|
||||
|
||||
// Perform all migrations.
|
||||
await MigrationService.run(migrations);
|
||||
|
||||
console.log('Settings created.');
|
||||
console.log('\nTalk is now installed!');
|
||||
|
||||
@@ -194,7 +201,7 @@ const performSetup = async () => {
|
||||
);
|
||||
};
|
||||
|
||||
// Start tthe setup process.
|
||||
// Start the setup process.
|
||||
performSetup()
|
||||
.then(() => {
|
||||
util.shutdown();
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Module dependencies.
|
||||
*/
|
||||
|
||||
const util = require('./util');
|
||||
const program = require('commander');
|
||||
const mongoose = require('../services/mongoose');
|
||||
const databaseVerifications = require('./verifications/database');
|
||||
|
||||
// Register the shutdown criteria.
|
||||
util.onshutdown([() => mongoose.disconnect()]);
|
||||
|
||||
async function database({ fix = false, limit = Infinity, batch = 1000 }) {
|
||||
try {
|
||||
for (const verification of databaseVerifications) {
|
||||
await verification({ fix, limit, batch });
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`Failed to process all the ${databaseVerifications.length} verifications`,
|
||||
err
|
||||
);
|
||||
util.shutdown(1);
|
||||
return;
|
||||
}
|
||||
|
||||
util.shutdown();
|
||||
}
|
||||
|
||||
//==============================================================================
|
||||
// Setting up the program command line arguments.
|
||||
//==============================================================================
|
||||
|
||||
program
|
||||
.command('db')
|
||||
.description('verifies the database integrity')
|
||||
.option('-f, --fix', 'fix the problems found with database inconsistencies')
|
||||
.option(
|
||||
'-l, --limit [size]',
|
||||
'limit the amount of documents to process in a single pass, this will ensure only a maximum number of batch operations are issued [default: inf]',
|
||||
parseInt
|
||||
)
|
||||
.option(
|
||||
'-b, --batch [size]',
|
||||
'batch size to process verifications and repairs of documents [default: 1000]',
|
||||
parseInt
|
||||
)
|
||||
.action(database);
|
||||
|
||||
program.parse(process.argv);
|
||||
|
||||
// If there is no command listed, output help.
|
||||
if (!process.argv.slice(2).length) {
|
||||
program.outputHelp();
|
||||
util.shutdown();
|
||||
}
|
||||
@@ -40,28 +40,6 @@ documents rather than performing a nice table alter. If the process crashes
|
||||
during the migration, simply re-run it. The migration operations are designed
|
||||
to act atomically, and be idempotent to documents already updated.
|
||||
|
||||
## Database Verifications
|
||||
|
||||
In `v3.*`, we introduced the concept of "verifying the database". Some of our
|
||||
operations update cached values that live along side the original document to
|
||||
improve performance. Running the cli command for verifying the database's cache
|
||||
ensures that all the cached values are up to date.
|
||||
|
||||
Running the following will start the database verification process:
|
||||
|
||||
```bash
|
||||
./bin/cli verify db --fix
|
||||
```
|
||||
You can notice the `--fix` option, without it, the tool should instead perform
|
||||
a dry run of the operations it intends to perform.
|
||||
{: .code-aside}
|
||||
|
||||
This process, like the migration process, should take some time to complete on
|
||||
large databases.
|
||||
|
||||
Once you have updated your databases, that's all you have to do! Talk should now
|
||||
function even better and faster with all the new features we poured into v4.0.0!
|
||||
|
||||
## Template Change
|
||||
|
||||
In `v4.0.0`, we introduced extensive support for compressing our javascript
|
||||
|
||||
@@ -1,65 +1,38 @@
|
||||
const CommentModel = require('../models/comment');
|
||||
|
||||
module.exports = {
|
||||
async up() {
|
||||
// Find all comments that have tags.
|
||||
let comments = await CommentModel.aggregate([
|
||||
{
|
||||
$match: {
|
||||
tags: {
|
||||
$exists: true,
|
||||
$ne: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
id: true,
|
||||
tags: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
// OLD
|
||||
//
|
||||
// [
|
||||
// {
|
||||
// name: 'OFF_TOPIC',
|
||||
// assigned_by: '',
|
||||
// created_at: new Date()
|
||||
// }
|
||||
// ]
|
||||
|
||||
// If no comments were found, nothing needs to be done!
|
||||
if (comments.length <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const updates = [];
|
||||
|
||||
// Loop over the comments retrieved, updating the tag structure.
|
||||
for (let { id, tags } of comments) {
|
||||
// OLD
|
||||
//
|
||||
// [
|
||||
// {
|
||||
// name: 'OFF_TOPIC',
|
||||
// assigned_by: '',
|
||||
// created_at: new Date()
|
||||
// }
|
||||
// ]
|
||||
|
||||
// NEW
|
||||
//
|
||||
// [
|
||||
// {
|
||||
// tag: {
|
||||
// name: 'OFF_TOPIC',
|
||||
// permissions: {
|
||||
// public: true,
|
||||
// self: false,
|
||||
// roles: []
|
||||
// },
|
||||
// models: ['COMMENTS'],
|
||||
// created_at: new Date()
|
||||
// },
|
||||
// assigned_by: '',
|
||||
// created_at: new Date()
|
||||
// }
|
||||
// ]
|
||||
|
||||
// Remap the tag structure.
|
||||
tags = tags.map(({ name, assigned_by, created_at }) => ({
|
||||
// NEW
|
||||
//
|
||||
// [
|
||||
// {
|
||||
// tag: {
|
||||
// name: 'OFF_TOPIC',
|
||||
// permissions: {
|
||||
// public: true,
|
||||
// self: false,
|
||||
// roles: []
|
||||
// },
|
||||
// models: ['COMMENTS'],
|
||||
// created_at: new Date()
|
||||
// },
|
||||
// assigned_by: '',
|
||||
// created_at: new Date()
|
||||
// }
|
||||
// ]
|
||||
const transformTags = ({ id, tags }) => ({
|
||||
query: { id },
|
||||
update: {
|
||||
$set: {
|
||||
tags: tags.map(({ name, assigned_by, created_at }) => ({
|
||||
tag: {
|
||||
name,
|
||||
permissions: {
|
||||
@@ -72,22 +45,34 @@ module.exports = {
|
||||
},
|
||||
assigned_by,
|
||||
created_at,
|
||||
}));
|
||||
})),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
updates.push({ query: { id }, update: { $set: { tags } } });
|
||||
}
|
||||
module.exports = {
|
||||
async up({ transformSingleWithCursor }) {
|
||||
// Find all comments that have tags.
|
||||
const cursor = CommentModel.collection.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
tags: {
|
||||
$exists: true,
|
||||
$ne: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
id: true,
|
||||
tags: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
{ allowDiskUse: true }
|
||||
);
|
||||
|
||||
if (updates.length > 0) {
|
||||
// Create a new batch operation.
|
||||
let batch = CommentModel.collection.initializeUnorderedBulkOp();
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
// Execute the batch operation.
|
||||
batch.find(query).updateOne(update);
|
||||
}
|
||||
|
||||
// Execute the batch update operation.
|
||||
await batch.execute();
|
||||
}
|
||||
await transformSingleWithCursor(cursor, transformTags, CommentModel);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -12,7 +12,7 @@ const mapping = {
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
async up() {
|
||||
async up({ processManyUpdates }) {
|
||||
const updates = [];
|
||||
for (const item_type in mapping) {
|
||||
const mappings = mapping[item_type];
|
||||
@@ -44,15 +44,7 @@ module.exports = {
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
// Setup the batch operation.
|
||||
const batch = ActionModel.collection.initializeUnorderedBulkOp();
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
batch.find(query).update(update);
|
||||
}
|
||||
|
||||
// Execute the batch update operation.
|
||||
await batch.execute();
|
||||
await processManyUpdates(ActionModel, updates);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,60 +1,132 @@
|
||||
const UserModel = require('../models/user');
|
||||
const merge = require('lodash/merge');
|
||||
|
||||
const getUserBatch = async () => {
|
||||
let query = {
|
||||
status: {
|
||||
$in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'],
|
||||
const transformUser = user => {
|
||||
const created_at = Date.now();
|
||||
|
||||
const { id, status, canEditName, suspension, disabled } = user;
|
||||
|
||||
let update = {
|
||||
$unset: {
|
||||
canEditName: '',
|
||||
suspension: '',
|
||||
disabled: '',
|
||||
},
|
||||
$set: {
|
||||
status: {
|
||||
// The username status is specific to each case.
|
||||
username: {
|
||||
history: [],
|
||||
},
|
||||
|
||||
// The user is not banned by default.
|
||||
banned: {
|
||||
status: false,
|
||||
history: [],
|
||||
},
|
||||
|
||||
// The user is not suspended by default.
|
||||
suspension: {
|
||||
until: null,
|
||||
history: [],
|
||||
},
|
||||
},
|
||||
updated_at: created_at,
|
||||
},
|
||||
};
|
||||
|
||||
// Find all the users that need migrating.
|
||||
return UserModel.collection.find(query);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
async up() {
|
||||
const created_at = Date.now();
|
||||
|
||||
// Get the first batch of users.
|
||||
let cursor = await getUserBatch();
|
||||
|
||||
const updates = [];
|
||||
while (await cursor.hasNext()) {
|
||||
const user = await cursor.next();
|
||||
|
||||
const { id, status, canEditName, suspension, disabled } = user;
|
||||
|
||||
let update = {
|
||||
$unset: {
|
||||
canEditName: '',
|
||||
suspension: '',
|
||||
disabled: '',
|
||||
if (disabled) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
banned: {
|
||||
status: true,
|
||||
history: [
|
||||
{
|
||||
status: true,
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
$set: {
|
||||
status: {
|
||||
// The username status is specific to each case.
|
||||
username: {
|
||||
history: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// The user is not banned by default.
|
||||
banned: {
|
||||
status: false,
|
||||
history: [],
|
||||
},
|
||||
// If the user has an "until" property of their suspension, then we need
|
||||
// to reflect that in the new status object.
|
||||
if (suspension && suspension.until !== null) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
suspension: {
|
||||
until: suspension.until,
|
||||
history: [
|
||||
{
|
||||
until: suspension.until,
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// The user is not suspended by default.
|
||||
suspension: {
|
||||
until: null,
|
||||
history: [],
|
||||
switch (status) {
|
||||
case 'ACTIVE':
|
||||
if (canEditName) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'UNSET',
|
||||
history: [
|
||||
{
|
||||
status: 'UNSET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
updated_at: created_at,
|
||||
},
|
||||
};
|
||||
|
||||
if (disabled) {
|
||||
});
|
||||
} else {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'SET',
|
||||
history: [
|
||||
{
|
||||
status: 'SET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'BANNED':
|
||||
if (canEditName) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'REJECTED',
|
||||
history: [
|
||||
{
|
||||
status: 'REJECTED',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} else {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
@@ -67,22 +139,11 @@ module.exports = {
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// If the user has an "until" property of their suspension, then we need
|
||||
// to reflect that in the new status object.
|
||||
if (suspension && suspension.until !== null) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
suspension: {
|
||||
until: suspension.until,
|
||||
username: {
|
||||
status: 'SET',
|
||||
history: [
|
||||
{
|
||||
until: suspension.until,
|
||||
status: 'SET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
@@ -91,138 +152,57 @@ module.exports = {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
switch (status) {
|
||||
case 'ACTIVE':
|
||||
if (canEditName) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'UNSET',
|
||||
history: [
|
||||
{
|
||||
status: 'UNSET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} else {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'SET',
|
||||
history: [
|
||||
{
|
||||
status: 'SET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'BANNED':
|
||||
if (canEditName) {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'REJECTED',
|
||||
history: [
|
||||
{
|
||||
status: 'REJECTED',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} else {
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
banned: {
|
||||
status: true,
|
||||
history: [
|
||||
{
|
||||
status: true,
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
username: {
|
||||
status: 'SET',
|
||||
history: [
|
||||
{
|
||||
status: 'SET',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 'PENDING':
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
break;
|
||||
case 'PENDING':
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'CHANGED',
|
||||
history: [
|
||||
{
|
||||
status: 'CHANGED',
|
||||
history: [
|
||||
{
|
||||
status: 'CHANGED',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
created_at,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
break;
|
||||
case 'APPROVED':
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
},
|
||||
},
|
||||
});
|
||||
break;
|
||||
case 'APPROVED':
|
||||
update = merge(update, {
|
||||
$set: {
|
||||
status: {
|
||||
username: {
|
||||
status: 'APPROVED',
|
||||
history: [
|
||||
{
|
||||
status: 'APPROVED',
|
||||
history: [
|
||||
{
|
||||
status: 'APPROVED',
|
||||
created_at,
|
||||
},
|
||||
],
|
||||
created_at,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new Error(`${status} is an invalid status`);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new Error(`${status} is an invalid status`);
|
||||
}
|
||||
|
||||
updates.push({ query: { id }, update });
|
||||
}
|
||||
return { query: { id }, update };
|
||||
};
|
||||
|
||||
if (updates.length > 0) {
|
||||
// Create a new batch operation.
|
||||
let bulk = UserModel.collection.initializeUnorderedBulkOp();
|
||||
module.exports = {
|
||||
async up({ transformSingleWithCursor }) {
|
||||
// Get the first batch of users.
|
||||
const cursor = UserModel.collection.find({
|
||||
status: {
|
||||
$in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'],
|
||||
},
|
||||
});
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
bulk.find(query).updateOne(update);
|
||||
}
|
||||
|
||||
// Execute the bulk update operation.
|
||||
await bulk.execute();
|
||||
}
|
||||
await transformSingleWithCursor(cursor, transformUser, UserModel);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -13,18 +13,16 @@ const findNewRole = roles => {
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
async up() {
|
||||
const cursor = await UserModel.collection.find({
|
||||
async up({ transformSingleWithCursor }) {
|
||||
const cursor = UserModel.collection.find({
|
||||
roles: {
|
||||
$exists: true,
|
||||
},
|
||||
});
|
||||
|
||||
const updates = [];
|
||||
while (await cursor.hasNext()) {
|
||||
const user = await cursor.next();
|
||||
|
||||
updates.push({
|
||||
await transformSingleWithCursor(
|
||||
cursor,
|
||||
user => ({
|
||||
query: {
|
||||
id: user.id,
|
||||
},
|
||||
@@ -38,19 +36,8 @@ module.exports = {
|
||||
roles: '',
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
// Create a new batch operation.
|
||||
const bulk = UserModel.collection.initializeUnorderedBulkOp();
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
bulk.find(query).updateOne(update);
|
||||
}
|
||||
|
||||
// Execute the bulk update operation.
|
||||
await bulk.execute();
|
||||
}
|
||||
}),
|
||||
UserModel
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
const ActionModel = require('../models/action');
|
||||
const UserModel = require('../models/user');
|
||||
const CommentModel = require('../models/comment');
|
||||
|
||||
module.exports = {
|
||||
async up({ transformSingleWithCursor }) {
|
||||
const models = [
|
||||
{ Model: CommentModel, item_type: 'COMMENTS' },
|
||||
{ Model: UserModel, item_type: 'USERS' },
|
||||
];
|
||||
for (const { Model, item_type } of models) {
|
||||
let cursor = ActionModel.collection.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
group_id: { $ne: null },
|
||||
item_type,
|
||||
},
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
// group unique documents by these properties, we are leveraging the
|
||||
// fact that each uuid is completely unique.
|
||||
_id: {
|
||||
item_id: '$item_id',
|
||||
action_type: '$action_type',
|
||||
group_id: '$group_id',
|
||||
},
|
||||
|
||||
// and sum up all actions matching the above grouping criteria
|
||||
count: {
|
||||
$sum: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
// suppress the _id field
|
||||
_id: false,
|
||||
|
||||
// map the fields from the _id grouping down a level
|
||||
item_id: '$_id.item_id',
|
||||
action_type: { $toLower: '$_id.action_type' },
|
||||
group_id: { $toLower: '$_id.group_id' },
|
||||
|
||||
// map the field directly
|
||||
count: '$count',
|
||||
},
|
||||
},
|
||||
],
|
||||
{ allowDiskUse: true }
|
||||
);
|
||||
|
||||
// Transform those documents.
|
||||
await transformSingleWithCursor(
|
||||
cursor,
|
||||
({ item_id, action_type, group_id, count }) => ({
|
||||
query: { id: item_id },
|
||||
update: {
|
||||
$set: {
|
||||
[`action_counts.${action_type}_${group_id}`]: count,
|
||||
},
|
||||
},
|
||||
}),
|
||||
Model
|
||||
);
|
||||
|
||||
// Secondly, we'll collect the group group id's (all the actions for a
|
||||
// specific action type) to update counts of.
|
||||
cursor = ActionModel.collection.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
item_type,
|
||||
},
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
// group unique documents by these properties, we are leveraging the
|
||||
// fact that each uuid is completely unique.
|
||||
_id: {
|
||||
item_id: '$item_id',
|
||||
action_type: '$action_type',
|
||||
},
|
||||
|
||||
// and sum up all actions matching the above grouping criteria
|
||||
count: {
|
||||
$sum: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$project: {
|
||||
// suppress the _id field
|
||||
_id: false,
|
||||
|
||||
// map the fields from the _id grouping down a level
|
||||
item_id: '$_id.item_id',
|
||||
action_type: { $toLower: '$_id.action_type' },
|
||||
|
||||
// map the field directly
|
||||
count: '$count',
|
||||
},
|
||||
},
|
||||
],
|
||||
{ allowDiskUse: true }
|
||||
);
|
||||
|
||||
// Transform those documents.
|
||||
await transformSingleWithCursor(
|
||||
cursor,
|
||||
({ item_id, action_type, count }) => ({
|
||||
query: { id: item_id },
|
||||
update: {
|
||||
$set: {
|
||||
[`action_counts.${action_type}`]: count,
|
||||
},
|
||||
},
|
||||
}),
|
||||
Model
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
const CommentModel = require('../models/comment');
|
||||
|
||||
const transformComments = ({ _id: parent_id, reply_count }) => ({
|
||||
query: { id: parent_id, reply_count: { $ne: reply_count } },
|
||||
update: { $set: { reply_count } },
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
async up({ transformSingleWithCursor }) {
|
||||
const cursor = CommentModel.collection.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
parent_id: { $ne: null },
|
||||
status: { $in: ['NONE', 'ACCEPTED'] },
|
||||
},
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$parent_id',
|
||||
reply_count: {
|
||||
$sum: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
{ allowDiskUse: true }
|
||||
);
|
||||
|
||||
// Transform those documents.
|
||||
await transformSingleWithCursor(cursor, transformComments, CommentModel);
|
||||
},
|
||||
};
|
||||
+6
-2
@@ -29,7 +29,7 @@
|
||||
},
|
||||
"talk": {
|
||||
"migration": {
|
||||
"minVersion": 1511801783
|
||||
"minVersion": 1516920160
|
||||
}
|
||||
},
|
||||
"repository": {
|
||||
@@ -222,6 +222,10 @@
|
||||
},
|
||||
"pre-commit": {
|
||||
"silent": false,
|
||||
"run": ["lint", "test:client", "test:server"]
|
||||
"run": [
|
||||
"lint",
|
||||
"test:client",
|
||||
"test:server"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
const debug = require('debug')('talk:services:migration');
|
||||
|
||||
/**
|
||||
* processUpdates processes batches of updates on the given model.
|
||||
*
|
||||
* @param {Object} model mongoose model that should perform the operations on
|
||||
* @param {Array<Object>} updates array of updates to execute
|
||||
*/
|
||||
const processUpdates = async (model, updates) => {
|
||||
// Create a new batch operation.
|
||||
const bulk = model.collection.initializeUnorderedBulkOp();
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
bulk.find(query).updateOne(update);
|
||||
}
|
||||
|
||||
// Execute the bulk update operation.
|
||||
await bulk.execute();
|
||||
};
|
||||
|
||||
const debugProcessStatistics = (count, totalCount) => {
|
||||
if (totalCount > 0) {
|
||||
debug(
|
||||
`processed ${(count / totalCount * 100).toFixed(
|
||||
2
|
||||
)}% (${count}/${totalCount}) updates`
|
||||
);
|
||||
} else {
|
||||
debug(`processed ${count} updates`);
|
||||
}
|
||||
};
|
||||
|
||||
const transformSingleWithCursor = ({
|
||||
queryBatchSize,
|
||||
updateBatchSize,
|
||||
}) => async (query, process, Model) => {
|
||||
debug('starting transform');
|
||||
|
||||
// We'll manage the updates that we store inside this object.
|
||||
let updates = [];
|
||||
|
||||
// Count the elements in the transformation.
|
||||
let totalCount = 0;
|
||||
try {
|
||||
totalCount = await query.count();
|
||||
} catch (err) {}
|
||||
|
||||
// First we'll collect all the individual actions with specific group id's.
|
||||
const cursor = await query.batchSize(queryBatchSize);
|
||||
|
||||
let count = 0;
|
||||
while (await cursor.hasNext()) {
|
||||
const result = await cursor.next();
|
||||
|
||||
const transformed = await process(result);
|
||||
if (transformed) {
|
||||
updates.push(transformed);
|
||||
}
|
||||
|
||||
if (updates.length > updateBatchSize) {
|
||||
// Process the updates.
|
||||
await processUpdates(Model, updates);
|
||||
count += updates.length;
|
||||
debugProcessStatistics(count, totalCount);
|
||||
|
||||
// Clear the updates array.
|
||||
updates = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
// Process the updates.
|
||||
await processUpdates(Model, updates);
|
||||
count += updates.length;
|
||||
debugProcessStatistics(count, totalCount);
|
||||
|
||||
// Clear the updates array.
|
||||
updates = [];
|
||||
}
|
||||
|
||||
debug('finished transform');
|
||||
};
|
||||
|
||||
/**
|
||||
* processManyUpdates processes batches of updates on many models with the given
|
||||
* model.
|
||||
*
|
||||
* @param {Object} model mongoose model that should perform the operations on
|
||||
* @param {Array<Object>} updates array of updates to execute
|
||||
*/
|
||||
const processManyUpdates = async (model, updates) => {
|
||||
// Create a new batch operation.
|
||||
const bulk = model.collection.initializeUnorderedBulkOp();
|
||||
|
||||
for (const { query, update } of updates) {
|
||||
bulk.find(query).update(update);
|
||||
}
|
||||
|
||||
// Execute the bulk update operation.
|
||||
await bulk.execute();
|
||||
};
|
||||
|
||||
module.exports = ctx => ({
|
||||
processManyUpdates,
|
||||
processUpdates,
|
||||
transformSingleWithCursor: transformSingleWithCursor(ctx),
|
||||
});
|
||||
@@ -1,17 +1,20 @@
|
||||
const MigrationModel = require('../models/migration');
|
||||
const MigrationModel = require('../../models/migration');
|
||||
const fs = require('fs');
|
||||
const ms = require('ms');
|
||||
const path = require('path');
|
||||
const Joi = require('joi');
|
||||
const debug = require('debug')('talk:services:migration');
|
||||
const sc = require('snake-case');
|
||||
const { talk: { migration: { minVersion } } } = require('../package.json');
|
||||
const helpers = require('./helpers');
|
||||
const { stripIndent } = require('common-tags');
|
||||
const { talk: { migration: { minVersion } } } = require('../../package.json');
|
||||
|
||||
const migrationTemplate = `module.exports = {
|
||||
async up() {
|
||||
|
||||
}
|
||||
};
|
||||
const migrationTemplate = stripIndent`
|
||||
module.exports = {
|
||||
async up({ queryBatchSize, updateBatchSize }) {
|
||||
|
||||
}
|
||||
};
|
||||
`;
|
||||
|
||||
class MigrationService {
|
||||
@@ -44,7 +47,7 @@ class MigrationService {
|
||||
static async listPending() {
|
||||
// Get all the migration files.
|
||||
let migrationFiles = fs.readdirSync(
|
||||
path.join(__dirname, '..', 'migrations')
|
||||
path.join(__dirname, '..', '..', 'migrations')
|
||||
);
|
||||
|
||||
// Ensure that all migrations follow this format.
|
||||
@@ -61,6 +64,7 @@ class MigrationService {
|
||||
|
||||
// Parse the migrations from the file listing.
|
||||
let migrations = migrationFiles
|
||||
.filter(filename => versionRe.test(filename))
|
||||
.map(filename => {
|
||||
// Parse the version from the filename.
|
||||
let matches = filename.match(versionRe);
|
||||
@@ -75,7 +79,7 @@ class MigrationService {
|
||||
}
|
||||
|
||||
// Read the migration from the filesystem.
|
||||
let migration = require(`../migrations/${filename}`);
|
||||
let migration = require(`../../migrations/${filename}`);
|
||||
Joi.assert(
|
||||
migration,
|
||||
migrationSchema,
|
||||
@@ -109,17 +113,26 @@ class MigrationService {
|
||||
*
|
||||
* @param {Array} migrations a list of migrations returned by `listPending`
|
||||
*/
|
||||
static async run(migrations) {
|
||||
static async run(
|
||||
migrations,
|
||||
{ queryBatchSize = 10000, updateBatchSize = 20000 } = {}
|
||||
) {
|
||||
if (migrations.length === 0) {
|
||||
console.log('No migrations to run!');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create the context helpers.
|
||||
const ctx = helpers({ queryBatchSize, updateBatchSize });
|
||||
|
||||
for (let { filename, version, migration } of migrations) {
|
||||
try {
|
||||
const startTime = new Date();
|
||||
console.log(`Starting migration ${filename}`);
|
||||
await migration.up();
|
||||
console.log(`Finished migration ${filename}`);
|
||||
await migration.up(ctx);
|
||||
const endTime = new Date();
|
||||
const totalTime = endTime.getTime() - startTime.getTime();
|
||||
console.log(`Finished migration ${filename} in ${ms(totalTime)}`);
|
||||
} catch (e) {
|
||||
console.error(`Migration ${filename} failed`);
|
||||
throw e;
|
||||
@@ -1,10 +1,14 @@
|
||||
const migration = require('../../../migrations/1510174676_user_status');
|
||||
const UserModel = require('../../../models/user');
|
||||
const helpers = require('../../../services/migration/helpers');
|
||||
|
||||
const chai = require('chai');
|
||||
chai.use(require('chai-datetime'));
|
||||
const { expect } = chai;
|
||||
|
||||
const performMigration = () =>
|
||||
migration.up(helpers({ queryBatchSize: 100, updateBatchSize: 100 }));
|
||||
|
||||
describe('migration.1510174676_user_status', () => {
|
||||
describe('active user', () => {
|
||||
beforeEach(async () => {
|
||||
@@ -24,7 +28,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
expect(user).to.have.property('canEditName', false);
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
@@ -54,7 +58,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
expect(user).to.have.property('canEditName', true);
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
@@ -85,7 +89,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
expect(user.canEditName).to.equal(true);
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
@@ -117,7 +121,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
expect(user.canEditName).to.equal(false);
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
@@ -153,7 +157,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
const until = user.suspension.until;
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
@@ -187,7 +191,7 @@ describe('migration.1510174676_user_status', () => {
|
||||
expect(user.status).to.equal('BANNED');
|
||||
|
||||
// Perform the migration.
|
||||
await migration.up();
|
||||
await performMigration();
|
||||
|
||||
user = await UserModel.collection.findOne({ id: '123' });
|
||||
|
||||
|
||||
@@ -55,8 +55,8 @@
|
||||
to-fast-properties "^2.0.0"
|
||||
|
||||
"@coralproject/eslint-config-talk@^0.1.0":
|
||||
version "0.1.0"
|
||||
resolved "https://registry.yarnpkg.com/@coralproject/eslint-config-talk/-/eslint-config-talk-0.1.0.tgz#3ddc5f6fb4362a1cd05a5fea56cdb3095afc8cc3"
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/@coralproject/eslint-config-talk/-/eslint-config-talk-0.1.1.tgz#71991b4937a3ffe657128d7f1170da4b5fb75c9e"
|
||||
dependencies:
|
||||
babel-eslint "^8.0.1"
|
||||
eslint-config-prettier "^2.9.0"
|
||||
@@ -2507,7 +2507,13 @@ dns-prefetch-control@0.1.0:
|
||||
version "0.1.0"
|
||||
resolved "https://registry.yarnpkg.com/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz#60ddb457774e178f1f9415f0cabb0e85b0b300b2"
|
||||
|
||||
doctrine@^2.0.0, doctrine@^2.0.2:
|
||||
doctrine@^2.0.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d"
|
||||
dependencies:
|
||||
esutils "^2.0.2"
|
||||
|
||||
doctrine@^2.0.2:
|
||||
version "2.0.2"
|
||||
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.2.tgz#68f96ce8efc56cc42651f1faadb4f175273b0075"
|
||||
dependencies:
|
||||
@@ -2746,7 +2752,7 @@ error-ex@^1.2.0, error-ex@^1.3.1:
|
||||
dependencies:
|
||||
is-arrayish "^0.2.1"
|
||||
|
||||
es-abstract@^1.4.3, es-abstract@^1.7.0:
|
||||
es-abstract@^1.4.3:
|
||||
version "1.9.0"
|
||||
resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.9.0.tgz#690829a07cae36b222e7fd9b75c0d0573eb25227"
|
||||
dependencies:
|
||||
@@ -2756,7 +2762,7 @@ es-abstract@^1.4.3, es-abstract@^1.7.0:
|
||||
is-callable "^1.1.3"
|
||||
is-regex "^1.0.4"
|
||||
|
||||
es-abstract@^1.6.1:
|
||||
es-abstract@^1.6.1, es-abstract@^1.7.0:
|
||||
version "1.10.0"
|
||||
resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.10.0.tgz#1ecb36c197842a00d8ee4c2dfd8646bb97d60864"
|
||||
dependencies:
|
||||
@@ -2873,8 +2879,8 @@ eslint-config-prettier@^2.9.0:
|
||||
get-stdin "^5.0.1"
|
||||
|
||||
eslint-plugin-jest@^21.6.1:
|
||||
version "21.6.1"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-21.6.1.tgz#adca015bbdb8d23b210438ff9e1cee1dd9ec35df"
|
||||
version "21.7.0"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-21.7.0.tgz#651f1c6ce999af3ac59ab8bf8a376d742fd0fc23"
|
||||
|
||||
eslint-plugin-mocha@^4.11.0:
|
||||
version "4.11.0"
|
||||
@@ -2883,8 +2889,8 @@ eslint-plugin-mocha@^4.11.0:
|
||||
ramda "^0.24.1"
|
||||
|
||||
eslint-plugin-prettier@^2.4.0:
|
||||
version "2.4.0"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-2.4.0.tgz#85cab0775c6d5e3344ef01e78d960f166fb93aae"
|
||||
version "2.5.0"
|
||||
resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-2.5.0.tgz#39a91dd7528eaf19cd42c0ee3f2c1f684606a05f"
|
||||
dependencies:
|
||||
fast-diff "^1.1.1"
|
||||
jest-docblock "^21.0.0"
|
||||
|
||||
Reference in New Issue
Block a user