wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-08-13 12:40:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

expanded on migration support

Browse Source
This commit is contained in:
Wyatt Johnson
2018-01-25 13:07:29 -07:00
parent ef7693afa5
commit e6f796e99c
8 changed files with 157 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
migrations/1496771633_tags.js
+32 -31
View File
@@ -1,34 +1,32 @@
const CommentModel = require('../models/comment');
const { processUpdates } = require('./utils');
module.exports = {
async up() {
async up({ queryBatchSize, updateBatchSize }) {
// Find all comments that have tags.
let comments = await CommentModel.aggregate([
{
$match: {
tags: {
$exists: true,
$ne: [],
const cursor = await CommentModel.collection
.aggregate([
{
$match: {
tags: {
$exists: true,
$ne: [],
},
},
},
},
{
$project: {
id: true,
tags: true,
{
$project: {
id: true,
tags: true,
},
},
},
]);
])
.batchSize(queryBatchSize);
// If no comments were found, nothing needs to be done!
if (comments.length <= 0) {
return;
}
let updates = [];
while (await cursor.hasNext()) {
let { id, tags } = await cursor.next();
const updates = [];
// Loop over the comments retrieved, updating the tag structure.
for (let { id, tags } of comments) {
// OLD
//
// [
@@ -75,19 +73,22 @@ module.exports = {
}));
updates.push({ query: { id }, update: { $set: { tags } } });
if (updates.length > updateBatchSize) {
// Process the updates.
await processUpdates(CommentModel, updates);
// Clear the updates array.
updates = [];
}
}
if (updates.length > 0) {
// Create a new batch operation.
let batch = CommentModel.collection.initializeUnorderedBulkOp();
// Process the updates.
await processUpdates(CommentModel, updates);
for (const { query, update } of updates) {
// Execute the batch operation.
batch.find(query).updateOne(update);
}
// Execute the batch update operation.
await batch.execute();
// Clear the updates array.
updates = [];
}
},
};
migrations/1507310316_flags.js
+2 -9
View File
@@ -1,4 +1,5 @@
const ActionModel = require('../models/action');
const { processUpdates } = require('./utils');
const mapping = {
COMMENTS: {
@@ -44,15 +45,7 @@ module.exports = {
}
if (updates.length > 0) {
// Setup the batch operation.
const batch = ActionModel.collection.initializeUnorderedBulkOp();
for (const { query, update } of updates) {
batch.find(query).update(update);
}
// Execute the batch update operation.
await batch.execute();
await processUpdates(ActionModel, updates);
}
},
};
migrations/1510174676_user_status.js
+12 -28
View File
@@ -1,35 +1,19 @@
const UserModel = require('../models/user');
const { processUpdates } = require('./utils');
const merge = require('lodash/merge');
const getUserBatch = async () => {
let query = {
status: {
$in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'],
},
};
// Find all the users that need migrating.
return UserModel.collection.find(query).batchSize(100);
};
const processUpdates = async updates => {
// Create a new batch operation.
let bulk = UserModel.collection.initializeUnorderedBulkOp();
for (const { query, update } of updates) {
bulk.find(query).updateOne(update);
}
// Execute the bulk update operation.
await bulk.execute();
};
module.exports = {
async up() {
async up({ queryBatchSize, updateBatchSize }) {
const created_at = Date.now();
// Get the first batch of users.
let cursor = await getUserBatch();
let cursor = await UserModel.collection
.find({
status: {
$in: ['ACTIVE', 'BANNED', 'PENDING', 'APPROVED'],
},
})
.batchSize(queryBatchSize);
let updates = [];
while (await cursor.hasNext()) {
@@ -225,9 +209,9 @@ module.exports = {
updates.push({ query: { id }, update });
// Process every 1000 users.
if (updates.length > 1000) {
if (updates.length > updateBatchSize) {
// Process the updates.
await processUpdates(updates);
await processUpdates(UserModel, updates);
// Clear the updates array.
updates = [];
@@ -236,7 +220,7 @@ module.exports = {
if (updates.length > 0) {
// Process the updates.
await processUpdates(updates);
await processUpdates(UserModel, updates);
// Clear the updates array.
updates = [];
migrations/1511801783_user_roles.js
+6 -17
View File
@@ -1,4 +1,5 @@
const UserModel = require('../models/user');
const { processUpdates } = require('./utils');
const findNewRole = roles => {
if (roles.includes('ADMIN')) {
@@ -12,27 +13,15 @@ const findNewRole = roles => {
return 'COMMENTER';
};
const processUpdates = async updates => {
// Create a new batch operation.
const bulk = UserModel.collection.initializeUnorderedBulkOp();
for (const { query, update } of updates) {
bulk.find(query).updateOne(update);
}
// Execute the bulk update operation.
await bulk.execute();
};
module.exports = {
async up() {
async up({ queryBatchSize, updateBatchSize }) {
const cursor = await UserModel.collection
.find({
roles: {
$exists: true,
},
})
.batchSize(100);
.batchSize(queryBatchSize);
let updates = [];
while (await cursor.hasNext()) {
@@ -54,9 +43,9 @@ module.exports = {
},
});
if (updates.length > 1000) {
if (updates.length > updateBatchSize) {
// Process the updates.
await processUpdates(updates);
await processUpdates(UserModel, updates);
// Clear the updates array.
updates = [];
@@ -65,7 +54,7 @@ module.exports = {
if (updates.length > 0) {
// Process the updates.
await processUpdates(updates);
await processUpdates(UserModel, updates);
// Clear the updates array.
updates = [];
migrations/utils.js
+19
View File
@@ -0,0 +1,19 @@
/**
* processUpdates processes batches of updates on the given model.
*
* @param {Object} model mongoose model that should perform the operations on
* @param {Array<Object>} updates array of updates to execute
*/
const processUpdates = async (model, updates) => {
// Create a new batch operation.
const bulk = model.collection.initializeUnorderedBulkOp();
for (const { query, update } of updates) {
bulk.find(query).updateOne(update);
}
// Execute the bulk update operation.
await bulk.execute();
};
module.exports = { processUpdates };