mirror of
https://github.com/wassname/talk.git
synced 2026-07-13 17:45:56 +08:00
removed unused code
This commit is contained in:
Wyatt Johnson
@@ -1,194 +0,0 @@
|
||||
const UserModel = require('../../../models/user');
|
||||
const CommentModel = require('../../../models/comment');
|
||||
const ActionsService = require('../../../services/actions');
|
||||
const { arrayJoinBy } = require('../../../graph/loaders/util');
|
||||
const { get } = require('lodash');
|
||||
const debug = require('debug')('talk:cli:verify');
|
||||
|
||||
const MODELS = [UserModel, CommentModel];
|
||||
|
||||
async function processBatch(Model, documents) {
|
||||
// Get an array of all the document id's.
|
||||
const documentIDs = documents.map(({ id }) => id);
|
||||
|
||||
// Store all the operations on this batch in this array that we'll return
|
||||
// later.
|
||||
const operations = [];
|
||||
|
||||
// Get the action summaries for this batch.
|
||||
const totalActionSummaries = await ActionsService.getActionSummaries(
|
||||
documentIDs
|
||||
).then(arrayJoinBy(documentIDs, 'item_id'));
|
||||
|
||||
// Iterate over the documents.
|
||||
for (let i = 0; i < documents.length; i++) {
|
||||
const document = documents[i];
|
||||
const actionSummaries = totalActionSummaries[i];
|
||||
|
||||
let ops = [];
|
||||
|
||||
for (const actionSummary of actionSummaries) {
|
||||
if (actionSummary.group_id === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// And we generate the group id.
|
||||
const ACTION_TYPE = actionSummary.action_type.toLowerCase();
|
||||
const GROUP_ID = actionSummary.group_id.toLowerCase();
|
||||
|
||||
if (GROUP_ID.length <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// And we add a new batch operation if the action summary is associated
|
||||
// with a group.
|
||||
const ACTION_COUNT_FIELD = `${ACTION_TYPE}_${GROUP_ID}`;
|
||||
|
||||
// Check that the action summaries match the cached counts.
|
||||
if (
|
||||
get(document, ['action_counts', ACTION_COUNT_FIELD]) !==
|
||||
actionSummary.count
|
||||
) {
|
||||
// Batch updates for those changes.
|
||||
ops.push({
|
||||
[`action_counts.${ACTION_COUNT_FIELD}`]: actionSummary.count,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Group all the action summaries together from all the different group
|
||||
// ids.
|
||||
const groupedActionSummaries = actionSummaries.reduce(
|
||||
(acc, actionSummary) => {
|
||||
// action_type is already snake cased (as it would have had to be when it
|
||||
// was inserted in the database).
|
||||
const ACTION_TYPE = actionSummary.action_type.toLowerCase();
|
||||
|
||||
if (!(ACTION_TYPE in acc)) {
|
||||
acc[ACTION_TYPE] = 0;
|
||||
}
|
||||
|
||||
acc[ACTION_TYPE] += actionSummary.count;
|
||||
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
for (const ACTION_COUNT_FIELD of Object.keys(groupedActionSummaries)) {
|
||||
const count = groupedActionSummaries[ACTION_COUNT_FIELD];
|
||||
|
||||
// Check that the action summaries match the cached counts.
|
||||
if (get(document, ['action_counts', ACTION_COUNT_FIELD]) !== count) {
|
||||
// Batch updates for those changes.
|
||||
ops.push({
|
||||
[`action_counts.${ACTION_COUNT_FIELD}`]: count,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// If this comment has action summaries that should be updated, then
|
||||
// perform an update!
|
||||
if (ops.length > 0) {
|
||||
operations.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
id: document.id,
|
||||
},
|
||||
update: {
|
||||
$set: Object.assign({}, ...ops),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return operations;
|
||||
}
|
||||
|
||||
module.exports = async ({ fix, batch }) => {
|
||||
for (const Model of MODELS) {
|
||||
const cursor = Model.collection
|
||||
.find({})
|
||||
.project({
|
||||
id: 1,
|
||||
action_counts: 1,
|
||||
})
|
||||
.sort({ created_at: 1 });
|
||||
|
||||
let operations = [];
|
||||
let documents = [];
|
||||
|
||||
// While there are documents to process.
|
||||
while (await cursor.hasNext()) {
|
||||
// Load the document.
|
||||
const document = await cursor.next();
|
||||
|
||||
// Push the document into the documents array.
|
||||
documents.push(document);
|
||||
|
||||
// Check to see if the length of the documents array requires us to
|
||||
// process it.
|
||||
if (documents.length > batch) {
|
||||
// Process this batch.
|
||||
let batchOperations = await processBatch(Model, documents);
|
||||
|
||||
// Push the batch operations into the model operations.
|
||||
operations.push(...batchOperations);
|
||||
|
||||
// Clear this batch contents.
|
||||
documents = [];
|
||||
}
|
||||
}
|
||||
|
||||
// Check to see if there are any documents left over.
|
||||
if (documents.length > 0) {
|
||||
// Process this batch.
|
||||
let batchOperations = await processBatch(Model, documents);
|
||||
|
||||
// Push the batch operations into the model operations.
|
||||
operations.push(...batchOperations);
|
||||
}
|
||||
|
||||
const OPERATIONS_LENGTH = operations.length;
|
||||
|
||||
console.log(
|
||||
`action_counts.js: ${OPERATIONS_LENGTH} ${
|
||||
Model.collection.name
|
||||
} need their action counts fixed.`
|
||||
);
|
||||
|
||||
// If fix was enabled, execute the batch writes.
|
||||
if (OPERATIONS_LENGTH > 0) {
|
||||
if (fix) {
|
||||
debug(
|
||||
`action_counts.js: fixing ${OPERATIONS_LENGTH} ${
|
||||
Model.collection.name
|
||||
}...`
|
||||
);
|
||||
|
||||
while (operations.length) {
|
||||
let result = await Model.collection.bulkWrite(
|
||||
operations.splice(0, batch)
|
||||
);
|
||||
|
||||
debug(
|
||||
`action_counts.js: fixed batch of ${result.modifiedCount} ${
|
||||
Model.collection.name
|
||||
}.`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`action_counts.js: applied all ${OPERATIONS_LENGTH} fixes to ${
|
||||
Model.collection.name
|
||||
}.`
|
||||
);
|
||||
} else {
|
||||
console.warn(
|
||||
'Skipping fixing, --fix was not enabled, pass --fix to fix these errors'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1,140 +0,0 @@
|
||||
const CommentModel = require('../../../models/comment');
|
||||
const { singleJoinBy } = require('../../../graph/loaders/util');
|
||||
const debug = require('debug')('talk:cli:verify');
|
||||
|
||||
const getBatch = async (limit, offset) =>
|
||||
CommentModel.find({})
|
||||
.select({ id: 1, action_counts: 1, reply_count: 1 })
|
||||
.limit(limit)
|
||||
.skip(offset)
|
||||
.sort('created_at');
|
||||
|
||||
module.exports = async ({ fix, limit, batch }) => {
|
||||
let operations = [];
|
||||
|
||||
// Count how many comments there are to process.
|
||||
const totalCount = await CommentModel.count();
|
||||
|
||||
let offset = 0;
|
||||
let comments = [];
|
||||
let commentIDs = [];
|
||||
|
||||
console.log(`Processing ${totalCount} comments in batches of ${limit}...`);
|
||||
|
||||
// Keep processing documents until there are is none left.
|
||||
while (offset < totalCount) {
|
||||
// Get a batch of comments.
|
||||
comments = await getBatch(batch, offset);
|
||||
commentIDs = comments.map(({ id }) => id);
|
||||
|
||||
// Get their reply counts.
|
||||
let allReplyCounts = await CommentModel.aggregate([
|
||||
{
|
||||
$match: {
|
||||
parent_id: {
|
||||
$in: commentIDs,
|
||||
},
|
||||
status: {
|
||||
$in: ['NONE', 'ACCEPTED'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$parent_id',
|
||||
count: {
|
||||
$sum: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
.then(singleJoinBy(commentIDs, '_id'))
|
||||
.then(results => results.map(result => (result ? result.count : 0)));
|
||||
|
||||
// Loop over the comments, with their action summaries.
|
||||
for (let i = 0; i < comments.length; i++) {
|
||||
let comment = comments[i];
|
||||
let replyCount = allReplyCounts[i];
|
||||
|
||||
// And check to see if the action summaries we just computed match what is
|
||||
// currently set for the comments.
|
||||
let commentOperations = [];
|
||||
|
||||
// If the reply count needs to be updated, then update it!
|
||||
if (comment.reply_count !== replyCount) {
|
||||
commentOperations.push({
|
||||
reply_count: replyCount,
|
||||
});
|
||||
}
|
||||
|
||||
// If this comment has action summaries that should be updated, then
|
||||
// perform an update!
|
||||
if (commentOperations.length > 0) {
|
||||
operations.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
id: comment.id,
|
||||
},
|
||||
update: {
|
||||
$set: Object.assign({}, ...commentOperations),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
debug(`Processed batch of ${comments.length} comments.`);
|
||||
|
||||
if (operations.length >= limit) {
|
||||
debug(
|
||||
`Queued operations are ${
|
||||
operations.length
|
||||
}, reached limit of ${limit}, not processing any more.`
|
||||
);
|
||||
|
||||
if (operations.length > limit) {
|
||||
debug(
|
||||
`${operations.length -
|
||||
limit} operations have been truncated to enforce the limit`
|
||||
);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
offset += batch;
|
||||
}
|
||||
|
||||
const OPERATIONS_LENGTH = operations.length;
|
||||
|
||||
if (limit < Infinity && offset + comments.length < totalCount) {
|
||||
console.log(
|
||||
`Processed ${offset +
|
||||
comments.length}/${totalCount} comments because we reached the update limit of ${limit}.`
|
||||
);
|
||||
} else {
|
||||
console.log(`Processed all ${totalCount} comments.`);
|
||||
}
|
||||
|
||||
console.log(`${OPERATIONS_LENGTH} documents need fixing.`);
|
||||
|
||||
// If fix was enabled, execute the batch writes.
|
||||
if (OPERATIONS_LENGTH > 0) {
|
||||
if (fix) {
|
||||
debug(`Fixing ${OPERATIONS_LENGTH} documents...`);
|
||||
|
||||
while (operations.length) {
|
||||
let batchOperations = operations.splice(0, batch);
|
||||
let result = await CommentModel.collection.bulkWrite(batchOperations);
|
||||
|
||||
debug(`Fixed batch of ${result.modifiedCount} documents.`);
|
||||
}
|
||||
|
||||
console.log(`Applied all ${OPERATIONS_LENGTH} fixes.`);
|
||||
} else {
|
||||
console.warn(
|
||||
'Skipping fixing, --fix was not enabled, pass --fix to fix these errors'
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1,10 +0,0 @@
|
||||
// This will import all the verifications that should be run by the:
|
||||
//
|
||||
// cli verify database
|
||||
//
|
||||
// command. They exist in the form:
|
||||
//
|
||||
// async ({fix = false, batch = 1000}) => {}
|
||||
//
|
||||
// where their options are derived.
|
||||
module.exports = [require('./comment_replies'), require('./action_counts')];
|
||||
Reference in New Issue
Block a user