feat: consolidated migrations for 6.0.0 release (#2831)

This commit is contained in:
Wyatt Johnson
2020-02-20 22:42:05 +00:00
committed by GitHub
parent 6865a71684
commit c2dadcc034
18 changed files with 216 additions and 1142 deletions
+21 -11
View File
@@ -1,10 +1,12 @@
import { merge } from "lodash";
import { Collection, IndexOptions } from "mongodb";
import { Collection, Db, IndexOptions } from "mongodb";
import now from "performance-now";
import { Writable } from "coral-common/types";
import logger from "coral-server/logger";
import collections from "../mongodb/collections";
type IndexType = 1 | -1 | "text";
export type IndexSpecification<T> = {
@@ -61,13 +63,9 @@ export function createIndexFactory<T>(
export function createConnectionOrderVariants<T>(
createIndexFn: IndexCreationFunction<T>,
variants: Array<IndexSpecification<T>>,
indexOptions: IndexOptions = { background: true }
variants: Array<IndexSpecification<T>>
) {
return async (
indexSpec: IndexSpecification<T>,
variantIndexOptions: IndexOptions = {}
) => {
return async (indexSpec: IndexSpecification<T>) => {
/**
* createIndexVariant will create a variant on the specified `indexSpec` that
* will include the new variation.
@@ -75,10 +73,7 @@ export function createConnectionOrderVariants<T>(
* @param variantSpec the spec that makes this variant different
*/
const createIndexVariant = (variantSpec: IndexSpecification<T>) =>
createIndexFn(
merge({}, indexSpec, variantSpec),
merge({}, indexOptions, variantIndexOptions)
);
createIndexFn(merge({}, indexSpec, variantSpec), { background: true });
// Create all the variants.
for (const variant of variants) {
@@ -86,3 +81,18 @@ export function createConnectionOrderVariants<T>(
}
};
}
export const createIndexesFactory = (mongo: Db) => ({
users: createIndexFactory(collections.users(mongo)),
invites: createIndexFactory(collections.invites(mongo)),
tenants: createIndexFactory(collections.tenants(mongo)),
comments: createIndexFactory(collections.comments(mongo)),
stories: createIndexFactory(collections.stories(mongo)),
commentActions: createIndexFactory(collections.commentActions(mongo)),
commentModerationActions: createIndexFactory(
collections.commentModerationActions(mongo)
),
queries: createIndexFactory(collections.queries(mongo)),
migrations: createIndexFactory(collections.migrations(mongo)),
sites: createIndexFactory(collections.sites(mongo)),
});
@@ -1,77 +0,0 @@
import { Db } from "mongodb";
import collections from "coral-server/services/mongodb/collections";
import { MigrationError } from "../error";
import Migration from "../migration";
export default class extends Migration {
public async down(mongo: Db, tenantID: string) {
// Remove the premod user status from all users on this Tenant.
const result = await collections.users(mongo).updateMany(
{
tenantID,
"status.premod": { $ne: null },
},
{
$unset: {
"status.premod": "",
},
}
);
this.log(tenantID).warn(
{ matchedCount: result.matchedCount, modifiedCount: result.matchedCount },
"cleared the premod status from users"
);
}
public async test(mongo: Db, tenantID: string) {
// Find all the users that still have premod status unset.
const cursor = collections
.users(mongo)
.find({
"status.premod": null,
tenantID,
})
.project({ id: 1 });
// Count them.
const users = await cursor.toArray();
const count = users.length;
if (count === 0) {
this.log(tenantID).info("all users migrated successfully");
return;
}
throw new MigrationError(
tenantID,
"found users that were not updated",
"users",
users.map(({ id }) => id)
);
}
public async up(mongo: Db, tenantID: string) {
// Migrate users to include the premod status.
const result = await collections.users(mongo).updateMany(
{
"status.premod": null,
tenantID,
},
{
$set: {
"status.premod": {
active: false,
history: [],
},
},
}
);
this.log(tenantID).info(
{ matchedCount: result.matchedCount, modifiedCount: result.matchedCount },
"updated user premod status"
);
}
}
@@ -1,372 +0,0 @@
import { Db, MongoError } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createConnectionOrderVariants, createIndexFactory } from "../indexing";
async function createMigrationRecordIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.migrations(mongo));
// UNIQUE { id }
await createIndex({ id: 1 }, { unique: true });
}
async function createUserIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.users(mongo));
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
// UNIQUE - PARTIAL { email }
await createIndex(
{ tenantID: 1, email: 1 },
{ unique: true, partialFilterExpression: { email: { $exists: true } } }
);
// UNIQUE { profiles.type, profiles.id }
await createIndex(
{ tenantID: 1, "profiles.type": 1, "profiles.id": 1 },
{
unique: true,
partialFilterExpression: { "profiles.id": { $exists: true } },
}
);
// { profiles }
await createIndex(
{ tenantID: 1, profiles: 1, email: 1 },
{
partialFilterExpression: { profiles: { $exists: true } },
background: true,
}
);
// TEXT { id, username, email, createdAt }
await createIndex(
{
tenantID: 1,
id: "text",
username: "text",
email: "text",
createdAt: -1,
},
{ background: true }
);
const variants = createConnectionOrderVariants(
createIndex,
[{ createdAt: -1 }],
{ background: true }
);
// User Connection pagination.
// { ...connectionParams }
await variants({
tenantID: 1,
});
// Role based User Connection pagination.
// { role, ...connectionParams }
await variants({
tenantID: 1,
role: 1,
});
// Suspension based User Connection pagination.
await variants({
tenantID: 1,
"status.suspension.history.from.start": 1,
"status.suspension.history.from.finish": 1,
});
// Ban based User Connection pagination.
await variants({
tenantID: 1,
"status.ban.active": 1,
});
}
async function createInviteIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.invites(mongo));
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
// UNIQUE { email }
await createIndex({ tenantID: 1, email: 1 }, { unique: true });
}
async function createTenantIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.tenants(mongo));
// UNIQUE { id }
await createIndex({ id: 1 }, { unique: true });
// UNIQUE { domain }
await createIndex({ domain: 1 }, { unique: true });
}
async function createStoryIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.stories(mongo));
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
// UNIQUE { url }
await createIndex({ tenantID: 1, url: 1 }, { unique: true });
// TEXT { $**, createdAt }
await createIndex(
{ tenantID: 1, "$**": "text", createdAt: -1 },
{ background: true }
);
const variants = createConnectionOrderVariants(
createIndex,
[{ createdAt: -1 }],
{ background: true }
);
// Story Connection pagination.
// { ...connectionParams }
await variants({
tenantID: 1,
});
// Closed At ordered Story Connection pagination.
// { closedAt, ...connectionParams }
await variants({
tenantID: 1,
closedAt: 1,
});
}
async function createStoryCountIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.stories(mongo));
// { createdAt }
await createIndex({ tenantID: 1, createdAt: 1 }, { background: true });
}
async function createQueriesIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.queries(mongo));
// UNIQUE { id }
await createIndex({ id: 1 }, { unique: true });
}
async function createCommentActionIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.commentActions(mongo));
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
// { actionType, commentID, userID }
await createIndex(
{ tenantID: 1, actionType: 1, commentID: 1, userID: 1 },
{ background: true }
);
const variants = createConnectionOrderVariants(
createIndex,
[{ createdAt: -1 }],
{ background: true }
);
// Connection pagination.
// { ...connectionParams }
await variants({
tenantID: 1,
actionType: 1,
commentID: 1,
});
}
async function createCommentModerationActionIndexes(mongo: Db) {
const createIndex = createIndexFactory(
collections.commentModerationActions(mongo)
);
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
const createVariants = createConnectionOrderVariants(createIndex, [
{ createdAt: -1 },
]);
// { moderatorID, ...connectionParams }
await createVariants({
moderatorID: 1,
});
}
async function createCommentIndexes(mongo: Db) {
const createIndex = createIndexFactory(collections.comments(mongo));
// UNIQUE { id }
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
// Facility for counting the tags against a story.
await createIndex(
{
tenantID: 1,
storyID: 1,
"tags.type": 1,
status: 1,
},
{
background: true,
partialFilterExpression: {
"tags.type": { $exists: true },
},
}
);
const streamVariants = createConnectionOrderVariants(createIndex, [
{ createdAt: -1 },
{ createdAt: 1 },
{ childCount: -1, createdAt: -1 },
{ "actionCounts.REACTION": -1, createdAt: -1 },
]);
// Story based Comment Connection pagination.
// { storyID, ...connectionParams }
await streamVariants({
tenantID: 1,
storyID: 1,
status: 1,
});
// Story + Reply based Comment Connection pagination.
// { storyID, ...connectionParams }
await streamVariants({
tenantID: 1,
storyID: 1,
parentID: 1,
status: 1,
});
// Author based Comment Connection pagination.
// { authorID, ...connectionParams }
await streamVariants({
tenantID: 1,
authorID: 1,
status: 1,
});
// Tag based Comment Connection pagination.
// { tags.type, ...connectionParams }
await streamVariants({
tenantID: 1,
storyID: 1,
"tags.type": 1,
});
const adminVariants = createConnectionOrderVariants(
createIndex,
[{ createdAt: -1 }, { createdAt: 1 }],
{ background: true }
);
// Moderation based Comment Connection pagination.
// { storyID, ...connectionParams }
await adminVariants({
tenantID: 1,
status: 1,
});
// Story based Comment Connection pagination that are flagged.
// { storyID, ...connectionParams }
await adminVariants({
tenantID: 1,
storyID: 1,
status: 1,
"actionCounts.FLAG": 1,
});
// Author based Comment Connection pagination.
// { authorID, ...connectionParams }
await adminVariants({
tenantID: 1,
authorID: 1,
});
}
type IndexCreationFunction = (mongo: Db) => Promise<void>;
const indexes: Array<[string, IndexCreationFunction]> = [
["migrations", createMigrationRecordIndexes],
["users", createUserIndexes],
["invites", createInviteIndexes],
["tenants", createTenantIndexes],
["comments", createCommentIndexes],
["stories", createStoryIndexes],
["stories", createStoryCountIndexes],
["commentActions", createCommentActionIndexes],
["commentModerationActions", createCommentModerationActionIndexes],
["queries", createQueriesIndexes],
];
export default class extends Migration {
/**
* ensureIndexes will ensure that all indexes have been created.
*
* @param mongo a MongoDB Database Connection
*/
private async ensureIndexes(mongo: Db) {
this.logger.info(
{ indexGroupCount: indexes.length },
"now ensuring indexes are created"
);
// For each of the index functions, call it.
for (const [indexGroup, indexFunction] of indexes) {
this.logger.info({ indexGroup }, "ensuring indexes are created");
await indexFunction(mongo);
this.logger.info({ indexGroup }, "indexes have been created");
}
this.logger.info("all indexes have been created");
}
public async indexes(mongo: Db) {
// Find all the collections that exist already.
const results = await mongo
.listCollections({}, { nameOnly: true })
.toArray();
const collectionNames = results
.filter(({ type }) => type === "collection")
.map(({ name }) => name);
// Drop existing indexes on managed collections so we can re-create them.
for (const collectionName in collections) {
if (!collections.hasOwnProperty(collectionName)) {
continue;
}
// Check to see if this collection exists.
if (!collectionNames.includes(collectionName)) {
continue;
}
try {
await mongo.collection(collectionName).dropIndexes();
} catch (err) {
if (err instanceof MongoError) {
// If we're dropping indexes on a collection that doesn't exist, then
// don't worry.
if (err.code === 26) {
continue;
}
}
throw err;
}
}
// Re-create the indexes for each collection now.
await this.ensureIndexes(mongo);
}
}
@@ -1,51 +0,0 @@
import { Db } from "mongodb";
import collections from "coral-server/services/mongodb/collections";
import Migration from "coral-server/services/migrate/migration";
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const result = await collections.users(mongo).updateMany(
{
tenantID,
moderatorNotes: null,
},
{
$set: {
moderatorNotes: [],
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.modifiedCount,
},
"added empty moderatorNotes array"
);
}
public async down(mongo: Db, tenantID: string) {
const result = await collections.users(mongo).updateMany(
{
tenantID,
moderatorNotes: {
$exists: true,
},
},
{
$unset: {
moderatorNotes: "",
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.matchedCount,
},
"removed moderatorNotes"
);
}
}
@@ -1,35 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createConnectionOrderVariants, createIndexFactory } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
const createIndex = createIndexFactory(collections.comments(mongo));
// Text searches.
await createIndex(
{
tenantID: 1,
"revisions.body": "text",
},
{ background: true }
);
const variants = createConnectionOrderVariants(createIndex, [
{ createdAt: -1 },
{ createdAt: 1 },
{ childCount: -1, createdAt: -1 },
{ "actionCounts.REACTION": -1, createdAt: -1 },
]);
// Story based Comment Connection pagination.
// { storyID, ...connectionParams }
await variants({
tenantID: 1,
storyID: 1,
});
}
}
@@ -1,38 +0,0 @@
import { Db } from "mongodb";
import { getDefaultStaffConfiguration } from "coral-server/models/tenant";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { MigrationError } from "../error";
export default class extends Migration {
private async getTenant(mongo: Db, id: string) {
const tenant = await collections.tenants(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant not found", "tenants", [id]);
}
return tenant;
}
public async test(mongo: Db, id: string) {
const tenant = await this.getTenant(mongo, id);
if (!tenant.staff) {
throw new MigrationError(id, "staff not set", "tenants", [id]);
}
}
public async up(mongo: Db, id: string) {
const tenant = await this.getTenant(mongo, id);
const bundle = this.i18n.getBundle(tenant.locale);
await collections.tenants(mongo).updateOne(
{ id, staff: null },
{
$set: {
staff: getDefaultStaffConfiguration(bundle),
},
}
);
}
}
@@ -1,56 +0,0 @@
import { Db } from "mongodb";
// Use the following collections reference to interact with specific
// collections.
import collections from "coral-server/services/mongodb/collections";
import Migration from "coral-server/services/migrate/migration";
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const result = await collections.tenants(mongo).updateOne(
{
id: tenantID,
newCommenters: null,
},
{
$set: {
newCommenters: {
premodEnabled: false,
approvedCommentsThreshold: 2,
},
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.matchedCount,
},
"added new commenters config"
);
}
public async down(mongo: Db, tenantID: string) {
const result = await collections.tenants(mongo).updateOne(
{
id: tenantID,
newCommenters: {
$exists: true,
},
},
{
$unset: {
newCommenters: "",
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.matchedCount,
},
"removed new commenters config"
);
}
}
@@ -1,19 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndexFactory } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
const createIndex = createIndexFactory(
collections.commentModerationActions(mongo)
);
await createIndex(
{ tenantID: 1, commentID: 1, createdAt: -1 },
{ background: true }
);
}
}
@@ -1,34 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndex } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
// Drop the old index.
await collections
.users(mongo)
.dropIndex("tenantID_1_profiles.type_1_profiles.id_1");
// Clean up the old users that have deleted their accounts.
await collections
.users(mongo)
.updateMany({ profiles: [] }, { $unset: { profiles: "" } });
// Add the new index.
await createIndex(
collections.users(mongo),
{
tenantID: 1,
"profiles.id": 1,
"profiles.type": 1,
},
{
unique: true,
partialFilterExpression: { profiles: { $exists: true } },
}
);
}
}
@@ -1,88 +0,0 @@
import { Db } from "mongodb";
import { Secret } from "coral-server/models/settings";
import { generateSecret, Tenant } from "coral-server/models/tenant";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { GQLTime } from "coral-server/graph/schema/__generated__/types";
import { MigrationError } from "../error";
interface OldTenant {
auth: {
integrations: {
sso: {
key?: string;
keyGeneratedAt?: GQLTime;
};
};
};
}
function isOldTenant(tenant: Tenant | OldTenant): tenant is OldTenant {
if ((tenant as Tenant).auth.integrations.sso.keys) {
return false;
}
return true;
}
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
// Get the Tenant so we can update it. We don't have to worry about two
// migration operations conflicting here because we will lock one instance
// to perform the operations.
const tenant = await collections
.tenants<OldTenant | Tenant>(mongo)
.findOne({ id: tenantID });
if (!tenant) {
throw new MigrationError(tenantID, "could not find tenant", "tenants", [
tenantID,
]);
}
if (!isOldTenant(tenant)) {
this.log(tenantID).info("tenant already has the new format for the keys");
return;
}
// Store the keys in an array.
const keys: Secret[] = [];
// Check to see if a key is set.
const sso = tenant.auth.integrations.sso;
if (sso.key && sso.keyGeneratedAt) {
// Create the new SSOKey based on this data.
const key = generateSecret("ssosec", sso.keyGeneratedAt);
// Set the secret of the sso key to the secret of the current set key.
key.secret = sso.key;
// Add this key to the set of keys.
keys.push(key);
} else {
throw new MigrationError(
tenantID,
"expected tenant to have at least one SSO key provided, found none",
"tenants",
[tenantID]
);
}
// Update the tenant with the new sso keys.
await collections.tenants(mongo).updateOne(
{ id: tenantID },
{
$set: {
"auth.integrations.sso.keys": keys,
},
$unset: {
"auth.integrations.sso.key": "",
"auth.integrations.sso.keyGeneratedAt": "",
},
}
);
}
}
@@ -1,122 +0,0 @@
import { DateTime } from "luxon";
import { Db } from "mongodb";
import { Secret } from "coral-server/models/settings";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { MigrationError } from "../error";
interface OldSSOKey {
kid: string;
secret?: string;
createdAt: Date;
deprecateAt?: Date;
deletedAt?: Date;
}
function isOldSSOKey(key: Secret | OldSSOKey): key is OldSSOKey {
if (!key) {
return true;
}
if ((key as Secret).inactiveAt) {
return false;
}
if ((key as Secret).rotatedAt) {
return false;
}
if (key.secret === "<deleted>") {
return false;
}
if ((key as OldSSOKey).deprecateAt) {
return true;
}
if ((key as OldSSOKey).deletedAt) {
return true;
}
return false;
}
interface OldTenant {
auth: {
integrations: {
sso: {
keys: OldSSOKey[];
};
};
};
}
export default class extends Migration {
public async down(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
// Transform the keys into the new format.
const keys: OldSSOKey[] = tenant.auth.integrations.sso.keys.map(
(key: OldSSOKey | Secret): OldSSOKey =>
!isOldSSOKey(key)
? {
kid: key.kid,
secret: key.secret === "<deleted>" ? undefined : key.secret,
createdAt: key.createdAt,
deprecateAt: key.inactiveAt,
}
: key
);
// Update the key to the new format.
await collections
.tenants<OldTenant>(mongo)
.updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
}
public async test(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
if (tenant.auth.integrations.sso.keys.some(key => isOldSSOKey(key))) {
throw new MigrationError(id, "old sso key was found", "tenants", [id]);
}
}
public async up(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
// Transform the keys into the new format.
const keys: Secret[] = tenant.auth.integrations.sso.keys.map(
(key): Secret => ({
kid: key.kid,
secret: key.secret || "<deleted>",
createdAt: key.createdAt,
inactiveAt: key.deprecateAt,
rotatedAt: key.deprecateAt
? DateTime.fromJSDate(key.deprecateAt)
.plus({ month: -1 })
.toJSDate()
: undefined,
})
);
// Update the key to the new format.
await collections
.tenants(mongo)
.updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
}
}
@@ -1,46 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndex } from "../indexing";
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const result = await collections.users(mongo).updateMany(
{
tenantID,
digests: {
$ne: [],
},
},
{
$set: {
hasDigests: true,
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.modifiedCount,
},
"added hasDigests flag"
);
}
public async indexes(mongo: Db) {
await createIndex(
collections.users(mongo),
{
tenantID: 1,
"notifications.digestFrequency": 1,
hasDigests: 1,
},
{
partialFilterExpression: { hasDigests: { $eq: true } },
background: true,
}
);
}
}
@@ -1,22 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndex } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
await createIndex(
collections.users(mongo),
{
tenantID: 1,
scheduledDeletionDate: 1,
},
{
partialFilterExpression: { scheduledDeletionDate: { $exists: true } },
background: true,
}
);
}
}
@@ -1,100 +0,0 @@
import { Db } from "mongodb";
import {
CommentStatusCounts,
createEmptyCommentStatusCounts,
} from "coral-server/models/comment";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { GQLCOMMENT_STATUS } from "coral-server/graph/schema/__generated__/types";
const BATCH_SIZE = 500;
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const cursor = collections
.comments<{
_id: string;
statuses: { status: GQLCOMMENT_STATUS; sum: number }[];
}>(mongo)
.aggregate([
// Find all comments written by this Tenant.
{
$match: { tenantID },
},
// Group each comment into it's authorID and status.
{
$group: {
_id: {
authorID: "$authorID",
status: "$status",
},
sum: { $sum: 1 },
},
},
// Group the documents from the previous stage into a count of each
// author's status counts.
{
$group: {
_id: "$_id.authorID",
statuses: {
$push: {
status: "$_id.status",
sum: "$sum",
},
},
},
},
]);
let updates: {
updateOne: {
filter: { tenantID: string; id: string };
update: { $set: { commentCounts: { status: CommentStatusCounts } } };
};
}[] = [];
while (await cursor.hasNext()) {
const doc = await cursor.next();
if (!doc) {
break;
}
// Reconstruct the comment status counts.
const statuses = createEmptyCommentStatusCounts();
for (const { status, sum } of doc.statuses) {
statuses[status] += sum;
}
// Push the update.
updates.push({
updateOne: {
filter: {
tenantID,
id: doc._id,
},
update: {
$set: {
commentCounts: {
status: statuses,
},
},
},
},
});
// Process updates if we are at the batch size.
if (updates.length >= BATCH_SIZE) {
await collections.users(mongo).bulkWrite(updates);
updates = [];
}
}
// Process any missed updates.
if (updates.length > 0) {
await collections.users(mongo).bulkWrite(updates);
updates = [];
}
}
}
@@ -1,21 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndexFactory } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
const createIndex = createIndexFactory(collections.stories(mongo));
// Create a partial sparse index on lastCommentedAt:
// tenantID: 1 <- ASC tenantIDs
// lastCommentedAt: -1 <- DESC dates (more recent to latest)
// ^ explained here: https://docs.mongodb.com/manual/core/index-compound/#sort-order
await createIndex(
{ tenantID: 1, lastCommentedAt: -1 },
{ partialFilterExpression: { lastCommentedAt: { $exists: true } } }
);
}
}
@@ -0,0 +1,195 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import {
createConnectionOrderVariants,
createIndexesFactory,
} from "../indexing";
function createIndexer(mongo: Db) {
// Get the indexing functions ready for each of the collections.
const index = createIndexesFactory(mongo);
// Get the variants ready for pagination indexing.
const variants = {
comments: {
admin: createConnectionOrderVariants(index.comments, [
{ createdAt: -1 },
{ createdAt: 1 },
]),
stream: createConnectionOrderVariants(index.comments, [
{ createdAt: -1 },
{ createdAt: 1 },
{ childCount: -1, createdAt: -1 },
{ "actionCounts.REACTION": -1, createdAt: -1 },
]),
},
users: createConnectionOrderVariants(index.users, [{ createdAt: -1 }]),
stories: createConnectionOrderVariants(index.stories, [{ createdAt: -1 }]),
commentActions: createConnectionOrderVariants(index.commentActions, [
{ createdAt: -1 },
]),
commentModerationActions: createConnectionOrderVariants(
index.commentModerationActions,
[{ createdAt: 1 }]
),
};
return { index, variants };
}
export default class extends Migration {
public async indexes(mongo: Db) {
// Create the indexer functions.
const { index, variants } = createIndexer(mongo);
// Comments
await index.comments({ tenantID: 1, id: 1 }, { unique: true });
await index.comments(
{ tenantID: 1, "revisions.body": "text" },
{ background: true }
);
await index.comments(
{
tenantID: 1,
storyID: 1,
"tags.type": 1,
status: 1,
},
{
partialFilterExpression: {
"tags.type": { $exists: true },
},
background: true,
}
);
await variants.comments.stream({ tenantID: 1, storyID: 1 });
await variants.comments.stream({ tenantID: 1, storyID: 1, status: 1 });
await variants.comments.stream({
tenantID: 1,
storyID: 1,
parentID: 1,
status: 1,
});
await variants.comments.stream({ tenantID: 1, authorID: 1, status: 1 });
await variants.comments.stream({ tenantID: 1, storyID: 1, "tags.type": 1 });
await variants.comments.admin({ tenantID: 1, status: 1 });
await variants.comments.admin({ tenantID: 1, authorID: 1 });
await variants.comments.admin({
tenantID: 1,
storyID: 1,
status: 1,
"actionCounts.FLAG": 1,
});
// Stories
await index.stories(
{ tenantID: 1, lastCommentedAt: -1 },
{
partialFilterExpression: { lastCommentedAt: { $exists: true } },
background: true,
}
);
await index.stories({ tenantID: 1, id: 1 }, { unique: true });
await index.stories({ tenantID: 1, url: 1 }, { unique: true });
await index.stories(
{ tenantID: 1, "$**": "text", createdAt: -1 },
{ background: true }
);
await index.stories({ tenantID: 1, createdAt: 1 }, { background: true });
await variants.stories({ tenantID: 1 });
await variants.stories({ tenantID: 1, closedAt: 1 });
// Comment Moderation Actions
await index.commentModerationActions(
{ tenantID: 1, id: 1 },
{ unique: true }
);
await index.commentModerationActions(
{ tenantID: 1, commentID: 1, createdAt: -1 },
{ background: true }
);
await variants.commentModerationActions({ tenantID: 1, moderatorID: 1 });
// Comment Actions
await index.commentActions({ tenantID: 1, id: 1 }, { unique: true });
await index.commentActions(
{ tenantID: 1, actionType: 1, commentID: 1, userID: 1 },
{ background: true }
);
await variants.commentActions({ tenantID: 1, actionType: 1, commentID: 1 });
// Users
await index.users({ tenantID: 1, id: 1 }, { unique: true });
await index.users(
{ tenantID: 1, email: 1 },
{ partialFilterExpression: { email: { $exists: true } }, unique: true }
);
await index.users(
{ tenantID: 1, profiles: 1, email: 1 },
{
partialFilterExpression: { profiles: { $exists: true } },
background: true,
}
);
await index.users(
{ tenantID: 1, scheduledDeletionDate: 1 },
{
partialFilterExpression: { scheduledDeletionDate: { $exists: true } },
background: true,
}
);
await index.users(
{ tenantID: 1, "notifications.digestFrequency": 1, hasDigests: 1 },
{
partialFilterExpression: { hasDigests: { $eq: true } },
background: true,
}
);
await index.users(
{ tenantID: 1, "profiles.id": 1, "profiles.type": 1 },
{
partialFilterExpression: { profiles: { $exists: true } },
unique: true,
}
);
await index.users(
{
tenantID: 1,
id: "text",
username: "text",
email: "text",
createdAt: -1,
},
{ background: true }
);
await variants.users({ tenantID: 1 });
await variants.users({ tenantID: 1, role: 1 });
await variants.users({
tenantID: 1,
"status.suspension.history.from.start": 1,
"status.suspension.history.from.finish": 1,
});
await variants.users({ tenantID: 1, "status.ban.active": 1 });
// Invites
await index.invites({ tenantID: 1, id: 1 }, { unique: true });
await index.invites({ tenantID: 1, email: 1 }, { unique: true });
// Migrations
await index.migrations({ id: 1 }, { unique: true });
// Tenants
await index.tenants({ id: 1 }, { unique: true });
await index.tenants({ domain: 1 }, { unique: true });
// Sites
await index.sites({ tenantID: 1, id: 1 }, { background: true });
await index.sites({ tenantID: 1, allowedOrigins: 1 }, { unique: true });
await index.sites({ tenantID: 1, name: 1 }, { background: true });
// Queries
await index.queries({ id: 1 }, { unique: true });
}
}
@@ -1,17 +0,0 @@
import { Db } from "mongodb";
// Use the following collections reference to interact with specific
// collections.
import collections from "coral-server/services/mongodb/collections";
import Migration from "coral-server/services/migrate/migration";
import { createIndexFactory } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
const createIndex = createIndexFactory(collections.sites(mongo));
await createIndex({ tenantID: 1, id: 1 }, { background: true });
await createIndex({ tenantID: 1, allowedOrigins: 1 }, { unique: true });
await createIndex({ tenantID: 1, name: 1 }, { background: true });
}
}
@@ -1,33 +0,0 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const result = await collections.users(mongo).updateMany(
{ tenantID, commentCounts: null },
{
$set: {
commentCounts: {
status: {
APPROVED: 0,
NONE: 0,
PREMOD: 0,
REJECTED: 0,
SYSTEM_WITHHELD: 0,
},
},
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.modifiedCount,
},
"applied fix to users"
);
}
}