From 674ab19d06ce162bdd9d115267084e1c99a00821 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 4 Dec 2017 09:05:40 -0700 Subject: [PATCH] Adjusted mongo indexes to support index scanning --- config.js | 4 ++ graph/typeDefs.graphql | 3 ++ models/comment.js | 55 +++++++++++++++++++-- models/user.js | 8 +++ plugin-api/beta/server/getReactionConfig.js | 22 +++++++++ services/mongoose.js | 6 ++- services/users.js | 22 +++++---- test/server/services/users.js | 14 ++---- 8 files changed, 109 insertions(+), 25 deletions(-) diff --git a/config.js b/config.js index 046b278e3..86cacaa83 100644 --- a/config.js +++ b/config.js @@ -23,6 +23,10 @@ const CONFIG = { APOLLO_ENGINE_KEY: process.env.APOLLO_ENGINE_KEY || null, ENABLE_TRACING: Boolean(process.env.APOLLO_ENGINE_KEY), + // When TRUE, it ensures that database indexes created in core will not add + // indexes. + CREATE_MONGO_INDEXES: process.env.DISABLE_CREATE_MONGO_INDEXES !== 'TRUE', + //------------------------------------------------------------------------------ // JWT based configuration //------------------------------------------------------------------------------ diff --git a/graph/typeDefs.graphql b/graph/typeDefs.graphql index 419cb50a4..1adbd2e47 100644 --- a/graph/typeDefs.graphql +++ b/graph/typeDefs.graphql @@ -40,6 +40,9 @@ enum USER_ROLES { # a staff of the site STAFF + + # a user without administrative roles. + COMMENTER } # Token is a personal access token associated with a given user. diff --git a/models/comment.js b/models/comment.js index e95ecc38e..8bfac1dd6 100644 --- a/models/comment.js +++ b/models/comment.js @@ -108,6 +108,21 @@ CommentSchema.index({ background: false }); +CommentSchema.index({ + 'status': 1, + 'created_at': 1, +}, { + background: true, +}); + +CommentSchema.index({ + 'status': 1, + 'created_at': 1, + 'asset_id': 1, +}, { + background: true, +}); + // Add an index that is optimized for sorting based on the action count data. CommentSchema.index({ 'created_at': 1, @@ -116,7 +131,40 @@ CommentSchema.index({ background: true, }); -// TODO: Evaluate adding indexes for each reaction as well. +CommentSchema.index({ + 'created_at': 1, + 'action_counts.flag': 1, + 'status': 1, +}, { + background: true, +}); + +// Add an index that is optimized for finding flagged comments. +CommentSchema.index({ + 'asset_id': 1, + 'created_at': 1, + 'action_counts.flag': 1, +}, { + background: true, +}); + +// Add an index for the reply sort. +CommentSchema.index({ + 'asset_id': 1, + 'created_at': -1, + 'reply_count': -1, +}, { + background: true, +}); + +// Optimize for tag searches/counts. +CommentSchema.index({ + 'asset_id': 1, + 'tags.tag.name': 1, + 'status': 1, +}, { + background: true, +}); // Add an index that is optimized for sorting based on the created_at timestamp // but also good at locating comments that have a specific asset id. @@ -136,7 +184,4 @@ CommentSchema.virtual('visible').get(function() { return ['ACCEPTED', 'NONE'].includes(this.status); }); -// Comment model. -const Comment = mongoose.model('Comment', CommentSchema); - -module.exports = Comment; +module.exports = mongoose.model('Comment', CommentSchema); diff --git a/models/user.js b/models/user.js index 98e004a38..f6797f94a 100644 --- a/models/user.js +++ b/models/user.js @@ -220,6 +220,14 @@ UserSchema.index({ background: false }); +UserSchema.index({ + 'lowercaseUsername': 1, + 'profiles.id': 1, + 'created_at': -1, +}, { + background: true, +}); + // This query is executed often, to count the number of flagged accounts with // usernames. UserSchema.index({ diff --git a/plugin-api/beta/server/getReactionConfig.js b/plugin-api/beta/server/getReactionConfig.js index 481990c16..52ebe49fe 100644 --- a/plugin-api/beta/server/getReactionConfig.js +++ b/plugin-api/beta/server/getReactionConfig.js @@ -1,10 +1,24 @@ const {SEARCH_OTHER_USERS} = require('../../../perms/constants'); const errors = require('../../../errors'); const pluralize = require('pluralize'); +const sc = require('snake-case'); +const CommentModel = require('../../../models/comment'); +const {CREATE_MONGO_INDEXES} = require('../../../config'); function getReactionConfig(reaction) { reaction = reaction.toLowerCase(); + if (CREATE_MONGO_INDEXES) { + + // Create the index on the comment model based on the reaction config. + CommentModel.collection.createIndex({ + created_at: 1, + [`action_counts.${sc(reaction)}`]: 1 + }, { + background: true, + }); + } + const reactionPlural = pluralize(reaction); const Reaction = reaction.charAt(0).toUpperCase() + reaction.slice(1); const REACTION = reaction.toUpperCase(); @@ -112,6 +126,14 @@ function getReactionConfig(reaction) { return { typeDefs, + schemas: ({CommentSchema}) => { + CommentSchema.index({ + 'created_at': 1, + [`action_counts.${sc(reaction)}`]: 1, + }, { + background: true, + }); + }, context: { Sort: () => ({ Comments: { diff --git a/services/mongoose.js b/services/mongoose.js index fb23a5fe1..94b357d19 100644 --- a/services/mongoose.js +++ b/services/mongoose.js @@ -5,7 +5,8 @@ const queryDebugger = require('debug')('talk:db:query'); const { MONGO_URL, - WEBPACK + WEBPACK, + CREATE_MONGO_INDEXES, } = require('../config'); // Loading the formatter from Mongoose: @@ -56,6 +57,9 @@ if (WEBPACK) { mongoose .connect(MONGO_URL, { useMongoClient: true, + config: { + autoIndex: CREATE_MONGO_INDEXES, + }, }) .then(() => { debug('connection established'); diff --git a/services/users.js b/services/users.js index f045e5916..52cd76dc2 100644 --- a/services/users.js +++ b/services/users.js @@ -701,7 +701,10 @@ class UsersService { return UserModel.find({}); } - value = escapeRegExp(value); + value = escapeRegExp(value).toLowerCase(); + + // Compile the prefix search regex. + const $regex = new RegExp(`^${value}`); return UserModel.find({ $or: [ @@ -709,8 +712,8 @@ class UsersService { // Search by a prefix match on the username. { 'lowercaseUsername': { - $regex: new RegExp(value.toLowerCase()) - } + $regex, + }, }, // Search by a prefix match on the email address. @@ -718,14 +721,13 @@ class UsersService { 'profiles': { $elemMatch: { id: { - $regex: new RegExp(value), - $options: 'i' + $regex, }, - provider: 'local' - } - } - } - ] + provider: 'local', + }, + }, + }, + ], }); } diff --git a/test/server/services/users.js b/test/server/services/users.js index 3c7a8f654..d137f0d57 100644 --- a/test/server/services/users.js +++ b/test/server/services/users.js @@ -186,18 +186,13 @@ describe('services.UsersService', () => { it('should match the search terms', async () => { const tests = [ - { - search: 'monster', - results: 1, - id: mockUsers[1].id, - }, { search: 'Stamp', results: 1, id: mockUsers[0].id, }, { - search: 'sockmonster@gmail.com', + search: 'sockmonster', results: 1, id: mockUsers[1].id, }, @@ -207,9 +202,10 @@ describe('services.UsersService', () => { id: mockUsers[2].id, }, { - search: 'gmail.com', - results: 3 - } + search: 'marvel', + results: 1, + id: mockUsers[2].id, + }, ]; for (const test of tests) {