wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-08-13 12:40:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

migration rewrite and removed verifications

Browse Source
This commit is contained in:
Wyatt Johnson
2018-01-25 16:12:26 -07:00
parent e8f73ddb87
commit 04390c5acd
17 changed files with 472 additions and 746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/cli
-1
View File
@@ -18,7 +18,6 @@ program
.command('token', 'work with the access tokens')
.command('users', 'work with the application auth')
.command('migration', 'provides utilities for migrating the database')
.command('verify', 'provides utilities for performing data verification')
.command(
'plugins',
'provides utilities for interacting with the plugin system'
bin/cli-verify
-58
View File
@@ -1,58 +0,0 @@
#!/usr/bin/env node
/**
* Module dependencies.
*/
const util = require('./util');
const program = require('commander');
const mongoose = require('../services/mongoose');
const databaseVerifications = require('./verifications/database');
// Register the shutdown criteria.
util.onshutdown([() => mongoose.disconnect()]);
async function database({ fix = false, limit = Infinity, batch = 1000 }) {
try {
for (const verification of databaseVerifications) {
await verification({ fix, limit, batch });
}
} catch (err) {
console.error(
`Failed to process all the ${databaseVerifications.length} verifications`,
err
);
util.shutdown(1);
return;
}
util.shutdown();
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.command('db')
.description('verifies the database integrity')
.option('-f, --fix', 'fix the problems found with database inconsistencies')
.option(
'-l, --limit [size]',
'limit the amount of documents to process in a single pass, this will ensure only a maximum number of batch operations are issued [default: inf]',
parseInt
)
.option(
'-b, --batch [size]',
'batch size to process verifications and repairs of documents [default: 1000]',
parseInt
)
.action(database);
program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/verifications/database/action_counts.js
-194
View File
@@ -1,194 +0,0 @@
const UserModel = require('../../../models/user');
const CommentModel = require('../../../models/comment');
const ActionsService = require('../../../services/actions');
const { arrayJoinBy } = require('../../../graph/loaders/util');
const { get } = require('lodash');
const debug = require('debug')('talk:cli:verify');
const MODELS = [UserModel, CommentModel];
async function processBatch(Model, documents) {
// Get an array of all the document id's.
const documentIDs = documents.map(({ id }) => id);
// Store all the operations on this batch in this array that we'll return
// later.
const operations = [];
// Get the action summaries for this batch.
const totalActionSummaries = await ActionsService.getActionSummaries(
documentIDs
).then(arrayJoinBy(documentIDs, 'item_id'));
// Iterate over the documents.
for (let i = 0; i < documents.length; i++) {
const document = documents[i];
const actionSummaries = totalActionSummaries[i];
let ops = [];
for (const actionSummary of actionSummaries) {
if (actionSummary.group_id === null) {
continue;
}
// And we generate the group id.
const ACTION_TYPE = actionSummary.action_type.toLowerCase();
const GROUP_ID = actionSummary.group_id.toLowerCase();
if (GROUP_ID.length <= 0) {
continue;
}
// And we add a new batch operation if the action summary is associated
// with a group.
const ACTION_COUNT_FIELD = `${ACTION_TYPE}_${GROUP_ID}`;
// Check that the action summaries match the cached counts.
if (
get(document, ['action_counts', ACTION_COUNT_FIELD]) !==
actionSummary.count
) {
// Batch updates for those changes.
ops.push({
[`action_counts.${ACTION_COUNT_FIELD}`]: actionSummary.count,
});
}
}
// Group all the action summaries together from all the different group
// ids.
const groupedActionSummaries = actionSummaries.reduce(
(acc, actionSummary) => {
// action_type is already snake cased (as it would have had to be when it
// was inserted in the database).
const ACTION_TYPE = actionSummary.action_type.toLowerCase();
if (!(ACTION_TYPE in acc)) {
acc[ACTION_TYPE] = 0;
}
acc[ACTION_TYPE] += actionSummary.count;
return acc;
},
{}
);
for (const ACTION_COUNT_FIELD of Object.keys(groupedActionSummaries)) {
const count = groupedActionSummaries[ACTION_COUNT_FIELD];
// Check that the action summaries match the cached counts.
if (get(document, ['action_counts', ACTION_COUNT_FIELD]) !== count) {
// Batch updates for those changes.
ops.push({
[`action_counts.${ACTION_COUNT_FIELD}`]: count,
});
}
}
// If this comment has action summaries that should be updated, then
// perform an update!
if (ops.length > 0) {
operations.push({
updateOne: {
filter: {
id: document.id,
},
update: {
$set: Object.assign({}, ...ops),
},
},
});
}
}
return operations;
}
module.exports = async ({ fix, batch }) => {
for (const Model of MODELS) {
const cursor = Model.collection
.find({})
.project({
id: 1,
action_counts: 1,
})
.sort({ created_at: 1 });
let operations = [];
let documents = [];
// While there are documents to process.
while (await cursor.hasNext()) {
// Load the document.
const document = await cursor.next();
// Push the document into the documents array.
documents.push(document);
// Check to see if the length of the documents array requires us to
// process it.
if (documents.length > batch) {
// Process this batch.
let batchOperations = await processBatch(Model, documents);
// Push the batch operations into the model operations.
operations.push(...batchOperations);
// Clear this batch contents.
documents = [];
}
}
// Check to see if there are any documents left over.
if (documents.length > 0) {
// Process this batch.
let batchOperations = await processBatch(Model, documents);
// Push the batch operations into the model operations.
operations.push(...batchOperations);
}
const OPERATIONS_LENGTH = operations.length;
console.log(
`action_counts.js: ${OPERATIONS_LENGTH} ${
Model.collection.name
} need their action counts fixed.`
);
// If fix was enabled, execute the batch writes.
if (OPERATIONS_LENGTH > 0) {
if (fix) {
debug(
`action_counts.js: fixing ${OPERATIONS_LENGTH} ${
Model.collection.name
}...`
);
while (operations.length) {
let result = await Model.collection.bulkWrite(
operations.splice(0, batch)
);
debug(
`action_counts.js: fixed batch of ${result.modifiedCount} ${
Model.collection.name
}.`
);
}
console.log(
`action_counts.js: applied all ${OPERATIONS_LENGTH} fixes to ${
Model.collection.name
}.`
);
} else {
console.warn(
'Skipping fixing, --fix was not enabled, pass --fix to fix these errors'
);
}
}
}
};
bin/verifications/database/comment_replies.js
-140
View File
@@ -1,140 +0,0 @@
const CommentModel = require('../../../models/comment');
const { singleJoinBy } = require('../../../graph/loaders/util');
const debug = require('debug')('talk:cli:verify');
const getBatch = async (limit, offset) =>
CommentModel.find({})
.select({ id: 1, action_counts: 1, reply_count: 1 })
.limit(limit)
.skip(offset)
.sort('created_at');
module.exports = async ({ fix, limit, batch }) => {
let operations = [];
// Count how many comments there are to process.
const totalCount = await CommentModel.count();
let offset = 0;
let comments = [];
let commentIDs = [];
console.log(`Processing ${totalCount} comments in batches of ${limit}...`);
// Keep processing documents until there are is none left.
while (offset < totalCount) {
// Get a batch of comments.
comments = await getBatch(batch, offset);
commentIDs = comments.map(({ id }) => id);
// Get their reply counts.
let allReplyCounts = await CommentModel.aggregate([
{
$match: {
parent_id: {
$in: commentIDs,
},
status: {
$in: ['NONE', 'ACCEPTED'],
},
},
},
{
$group: {
_id: '$parent_id',
count: {
$sum: 1,
},
},
},
])
.then(singleJoinBy(commentIDs, '_id'))
.then(results => results.map(result => (result ? result.count : 0)));
// Loop over the comments, with their action summaries.
for (let i = 0; i < comments.length; i++) {
let comment = comments[i];
let replyCount = allReplyCounts[i];
// And check to see if the action summaries we just computed match what is
// currently set for the comments.
let commentOperations = [];
// If the reply count needs to be updated, then update it!
if (comment.reply_count !== replyCount) {
commentOperations.push({
reply_count: replyCount,
});
}
// If this comment has action summaries that should be updated, then
// perform an update!
if (commentOperations.length > 0) {
operations.push({
updateOne: {
filter: {
id: comment.id,
},
update: {
$set: Object.assign({}, ...commentOperations),
},
},
});
}
}
debug(`Processed batch of ${comments.length} comments.`);
if (operations.length >= limit) {
debug(
`Queued operations are ${
operations.length
}, reached limit of ${limit}, not processing any more.`
);
if (operations.length > limit) {
debug(
`${operations.length -
limit} operations have been truncated to enforce the limit`
);
}
break;
}
offset += batch;
}
const OPERATIONS_LENGTH = operations.length;
if (limit < Infinity && offset + comments.length < totalCount) {
console.log(
`Processed ${offset +
comments.length}/${totalCount} comments because we reached the update limit of ${limit}.`
);
} else {
console.log(`Processed all ${totalCount} comments.`);
}
console.log(`${OPERATIONS_LENGTH} documents need fixing.`);
// If fix was enabled, execute the batch writes.
if (OPERATIONS_LENGTH > 0) {
if (fix) {
debug(`Fixing ${OPERATIONS_LENGTH} documents...`);
while (operations.length) {
let batchOperations = operations.splice(0, batch);
let result = await CommentModel.collection.bulkWrite(batchOperations);
debug(`Fixed batch of ${result.modifiedCount} documents.`);
}
console.log(`Applied all ${OPERATIONS_LENGTH} fixes.`);
} else {
console.warn(
'Skipping fixing, --fix was not enabled, pass --fix to fix these errors'
);
}
}
};
bin/verifications/database/index.js
-10
View File
@@ -1,10 +0,0 @@
// This will import all the verifications that should be run by the:
//
// cli verify database
//
// command. They exist in the form:
//
// async ({fix = false, batch = 1000}) => {}
//
// where their options are derived.
module.exports = [require('./comment_replies'), require('./action_counts')];