mirror of
https://github.com/wassname/talk.git
synced 2026-06-27 18:45:03 +08:00
[next] MongoDB Indexes (#2142)
* feat: added mongo indexing support * fix: fixed typescript issue * chore: better types * fix: revert debug stuff * fix: addressed ts error * feat: added config option to disable auto-indexing * chore: reordered imports * refactor: cleaned up some filepaths
This commit is contained in:
Wyatt Johnson
Generated
+14
-33
@@ -7143,7 +7143,7 @@
|
||||
},
|
||||
"colors": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
|
||||
"integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=",
|
||||
"dev": true
|
||||
},
|
||||
@@ -7951,7 +7951,7 @@
|
||||
},
|
||||
"css-color-names": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "http://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
|
||||
"resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
|
||||
"integrity": "sha1-gIrcLnnPhHOAabZGyyDsJ762KeA=",
|
||||
"dev": true
|
||||
},
|
||||
@@ -11378,7 +11378,7 @@
|
||||
"integrity": "sha1-ETOUSrJHeINHOZVZaIPg05z4hc8=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"intl-pluralrules": "github:projectfluent/IntlPluralRules#module"
|
||||
"intl-pluralrules": "github:projectfluent/IntlPluralRules#94cb0fa1c23ad943bc5aafef43cea132fa51d68b"
|
||||
}
|
||||
},
|
||||
"fluent-langneg": {
|
||||
@@ -11672,8 +11672,7 @@
|
||||
"ansi-regex": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
|
||||
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
|
||||
"optional": true
|
||||
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
@@ -11694,14 +11693,12 @@
|
||||
"balanced-match": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
|
||||
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
|
||||
"optional": true
|
||||
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
@@ -11716,20 +11713,17 @@
|
||||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
|
||||
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
|
||||
"optional": true
|
||||
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
|
||||
"optional": true
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
|
||||
},
|
||||
"console-control-strings": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
|
||||
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
|
||||
"optional": true
|
||||
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
@@ -11846,8 +11840,7 @@
|
||||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
|
||||
"optional": true
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.5",
|
||||
@@ -11859,7 +11852,6 @@
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
|
||||
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"number-is-nan": "^1.0.0"
|
||||
}
|
||||
@@ -11874,7 +11866,6 @@
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
@@ -11882,14 +11873,12 @@
|
||||
"minimist": {
|
||||
"version": "0.0.8",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
|
||||
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
|
||||
"optional": true
|
||||
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.2.4",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.2.4.tgz",
|
||||
"integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.1",
|
||||
"yallist": "^3.0.0"
|
||||
@@ -11908,7 +11897,6 @@
|
||||
"version": "0.5.1",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
|
||||
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"minimist": "0.0.8"
|
||||
}
|
||||
@@ -11989,8 +11977,7 @@
|
||||
"number-is-nan": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
|
||||
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
|
||||
"optional": true
|
||||
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
@@ -12002,7 +11989,6 @@
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
@@ -12088,8 +12074,7 @@
|
||||
"safe-buffer": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
|
||||
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==",
|
||||
"optional": true
|
||||
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
@@ -12125,7 +12110,6 @@
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
|
||||
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"code-point-at": "^1.0.0",
|
||||
"is-fullwidth-code-point": "^1.0.0",
|
||||
@@ -12145,7 +12129,6 @@
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"ansi-regex": "^2.0.0"
|
||||
}
|
||||
@@ -12189,14 +12172,12 @@
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
|
||||
"optional": true
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz",
|
||||
"integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=",
|
||||
"optional": true
|
||||
"integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k="
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
+2
-2
@@ -30,8 +30,8 @@
|
||||
"generate:relay-admin": "ts-node ./scripts/compileRelay --src ./src/core/client/admin --schema tenant",
|
||||
"generate:schema": "node ./scripts/generateSchemaTypes.js",
|
||||
"docz": "docz",
|
||||
"start": "node dist/index.js",
|
||||
"start:development": "CONCURRENCY=2 TS_NODE_PROJECT=./src/tsconfig.json node -r ts-node/register -r tsconfig-paths/register ./src/index.ts",
|
||||
"start": "NODE_ENV=production node dist/index.js",
|
||||
"start:development": "NODE_ENV=development CONCURRENCY=2 TS_NODE_PROJECT=./src/tsconfig.json node -r ts-node/register -r tsconfig-paths/register ./src/index.ts",
|
||||
"start:webpackDevServer": "ts-node ./scripts/start.ts",
|
||||
"lint": "npm-run-all --parallel lint:* tscheck:*",
|
||||
"lint:server": "tslint --project ./src/tsconfig.json",
|
||||
|
||||
@@ -34,7 +34,7 @@ async function main() {
|
||||
config: {
|
||||
contextType: "TenantContext",
|
||||
importStatements: [
|
||||
'import { Cursor } from "talk-server/models/connection";',
|
||||
'import { Cursor } from "talk-server/models/helpers/connection";',
|
||||
'import TenantContext from "talk-server/graph/tenant/context";',
|
||||
],
|
||||
customScalarType: { Cursor: "Cursor", Time: "Date" },
|
||||
|
||||
@@ -151,7 +151,11 @@ export async function findOrCreateOIDCUser(
|
||||
};
|
||||
|
||||
// Try to lookup user given their id provided in the `sub` claim.
|
||||
let user = await retrieveUserWithProfile(db, tenant.id, profile);
|
||||
let user = await retrieveUserWithProfile(db, tenant.id, {
|
||||
// NOTE: (wyattjoh) as the current requirements do not allow multiple OIDC integrations, we are only getting the profile based on the OIDC provider.
|
||||
type: "oidc",
|
||||
id: sub,
|
||||
});
|
||||
if (!user) {
|
||||
if (!integration.allowRegistration) {
|
||||
// Registration is disabled, so we can't create the user user here.
|
||||
|
||||
@@ -140,6 +140,13 @@ const config = convict({
|
||||
env: "DISABLE_TENANT_CACHING",
|
||||
arg: "disableTenantCaching",
|
||||
},
|
||||
disable_mongodb_autoindexing: {
|
||||
doc: "Disables the creation of new MongoDB indexes",
|
||||
format: Boolean,
|
||||
default: false,
|
||||
env: "DISABLE_MONGODB_AUTOINDEXING",
|
||||
arg: "disableMongoDBAutoindexing",
|
||||
},
|
||||
});
|
||||
|
||||
export type Config = typeof config;
|
||||
|
||||
@@ -2,7 +2,7 @@ import { GraphQLScalarType } from "graphql";
|
||||
import { Kind } from "graphql/language";
|
||||
import { DateTime } from "luxon";
|
||||
|
||||
import { Cursor } from "talk-server/models/connection";
|
||||
import { Cursor } from "talk-server/models/helpers/connection";
|
||||
|
||||
function parseIntegerCursor(value: string): number | null {
|
||||
try {
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
retrieveCommentUserConnection,
|
||||
retrieveManyComments,
|
||||
} from "talk-server/models/comment";
|
||||
import { Connection } from "talk-server/models/connection";
|
||||
import { Connection } from "talk-server/models/helpers/connection";
|
||||
import { retrieveSharedModerationQueueQueuesCounts } from "talk-server/models/story/counts/shared";
|
||||
|
||||
import { SingletonResolver } from "./util";
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
import { decodeActionCounts } from "talk-server/models/action/comment";
|
||||
import * as comment from "talk-server/models/comment";
|
||||
import { getLatestRevision } from "talk-server/models/comment";
|
||||
import { createConnection } from "talk-server/models/connection";
|
||||
import { createConnection } from "talk-server/models/helpers/connection";
|
||||
|
||||
import TenantContext from "../context";
|
||||
import { getURLWithCommentID } from "./util";
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
RejectCommentPayloadToModerationQueuesResolver,
|
||||
} from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import { CommentConnectionInput } from "talk-server/models/comment";
|
||||
import { FilterQuery } from "talk-server/models/query";
|
||||
import { FilterQuery } from "talk-server/models/helpers/query";
|
||||
import {
|
||||
CommentModerationCountsPerQueue,
|
||||
Story,
|
||||
|
||||
@@ -1547,36 +1547,6 @@ type Story {
|
||||
createdAt: Time!
|
||||
}
|
||||
|
||||
"""
|
||||
StoryEdge represents a unique Story in a StoryConnection.
|
||||
"""
|
||||
type StoryEdge {
|
||||
"""
|
||||
node is the Story for this edge.
|
||||
"""
|
||||
node: Story!
|
||||
|
||||
"""
|
||||
|
||||
"""
|
||||
cursor: Cursor!
|
||||
}
|
||||
|
||||
"""
|
||||
StoriesConnection represents a subset of a Story list.
|
||||
"""
|
||||
type StoriesConnection {
|
||||
"""
|
||||
edges are a subset of StoryEdge's.
|
||||
"""
|
||||
edges: [StoryEdge!]!
|
||||
|
||||
"""
|
||||
pageInfo is
|
||||
"""
|
||||
pageInfo: PageInfo!
|
||||
}
|
||||
|
||||
################################################################################
|
||||
## CommentsFilterInput
|
||||
################################################################################
|
||||
|
||||
@@ -15,6 +15,7 @@ import logger from "talk-server/logger";
|
||||
import { createQueue, TaskQueue } from "talk-server/queue";
|
||||
import { createJWTSigningConfig } from "talk-server/services/jwt";
|
||||
import { createMongoDB } from "talk-server/services/mongodb";
|
||||
import { ensureIndexes } from "talk-server/services/mongodb/indexes";
|
||||
import { AugmentedRedis, createRedisClient } from "talk-server/services/redis";
|
||||
import TenantCache from "talk-server/services/tenant/cache";
|
||||
|
||||
@@ -22,6 +23,10 @@ export interface ServerOptions {
|
||||
config?: Config;
|
||||
}
|
||||
|
||||
export interface ServerConnectOptions {
|
||||
isWorker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Server provides an interface to create, start, and manage a Talk Server.
|
||||
*/
|
||||
@@ -74,7 +79,7 @@ class Server {
|
||||
};
|
||||
}
|
||||
|
||||
public async connect() {
|
||||
public async connect({ isWorker }: ServerConnectOptions = {}) {
|
||||
// Guard against double connecting.
|
||||
if (this.connected) {
|
||||
throw new Error("server has already connected");
|
||||
@@ -84,6 +89,13 @@ class Server {
|
||||
// Setup MongoDB.
|
||||
this.mongo = await createMongoDB(config);
|
||||
|
||||
// If the instance being connected is not a worker process, then create the
|
||||
// database indexes if it isn't disabled.
|
||||
if (!isWorker && !this.config.get("disable_mongodb_autoindexing")) {
|
||||
// Setup the database indexes.
|
||||
await ensureIndexes(this.mongo);
|
||||
}
|
||||
|
||||
// Setup Redis.
|
||||
this.redis = createRedisClient(config);
|
||||
|
||||
|
||||
@@ -11,7 +11,10 @@ import {
|
||||
GQLCOMMENT_FLAG_REASON,
|
||||
GQLCOMMENT_FLAG_REPORTED_REASON,
|
||||
} from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import { FilterQuery } from "talk-server/models/query";
|
||||
import {
|
||||
createIndexFactory,
|
||||
FilterQuery,
|
||||
} from "talk-server/models/helpers/query";
|
||||
import { TenantResource } from "talk-server/models/tenant";
|
||||
|
||||
function collection(db: Db) {
|
||||
@@ -103,6 +106,16 @@ export interface CommentAction extends TenantResource {
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export async function createCommentActionIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
|
||||
|
||||
// { actionType, commentID }
|
||||
await createIndex({ tenantID: 1, actionType: 1, commentID: 1, userID: 1 });
|
||||
}
|
||||
|
||||
const ActionSchema = [
|
||||
// Flags
|
||||
{
|
||||
@@ -245,9 +258,9 @@ export async function retrieveUserAction(
|
||||
) {
|
||||
return collection(mongo).findOne({
|
||||
tenantID,
|
||||
actionType,
|
||||
commentID,
|
||||
userID,
|
||||
actionType,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -8,8 +8,11 @@ import {
|
||||
ConnectionInput,
|
||||
getPageInfo,
|
||||
nodesToEdges,
|
||||
} from "talk-server/models/connection";
|
||||
import Query from "talk-server/models/query";
|
||||
} from "talk-server/models/helpers/connection";
|
||||
import Query, {
|
||||
createConnectionOrderVariants,
|
||||
createIndexFactory,
|
||||
} from "talk-server/models/helpers/query";
|
||||
import { TenantResource } from "talk-server/models/tenant";
|
||||
|
||||
function collection(db: Db) {
|
||||
@@ -18,6 +21,22 @@ function collection(db: Db) {
|
||||
);
|
||||
}
|
||||
|
||||
export async function createCommentModerationActionIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
|
||||
|
||||
const createVariants = createConnectionOrderVariants<
|
||||
Readonly<CommentModerationAction>
|
||||
>([{ createdAt: -1 }]);
|
||||
|
||||
// { moderatorID, ...connectionParams }
|
||||
await createVariants(createIndex, {
|
||||
moderatorID: 1,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* CommentModerationAction stores information around a moderation action that
|
||||
* was created for a given Comment Revision.
|
||||
|
||||
@@ -19,8 +19,11 @@ import {
|
||||
nodesToEdges,
|
||||
NodeToCursorTransformer,
|
||||
OrderedConnectionInput,
|
||||
} from "talk-server/models/connection";
|
||||
import Query from "talk-server/models/query";
|
||||
} from "talk-server/models/helpers/connection";
|
||||
import Query, {
|
||||
createConnectionOrderVariants,
|
||||
createIndexFactory,
|
||||
} from "talk-server/models/helpers/query";
|
||||
import { TenantResource } from "talk-server/models/tenant";
|
||||
|
||||
function collection(mongo: Db) {
|
||||
@@ -137,6 +140,54 @@ export interface Comment extends TenantResource {
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export async function createCommentIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
|
||||
|
||||
const variants = createConnectionOrderVariants<Readonly<Comment>>([
|
||||
{ createdAt: -1 },
|
||||
{ createdAt: 1 },
|
||||
{ replyCount: -1, createdAt: -1 },
|
||||
{ "actionCounts.REACTION": -1, createdAt: -1 },
|
||||
]);
|
||||
|
||||
// Story based Comment Connection pagination.
|
||||
// { storyID, ...connectionParams }
|
||||
await variants(createIndex, {
|
||||
tenantID: 1,
|
||||
storyID: 1,
|
||||
status: 1,
|
||||
});
|
||||
|
||||
// Story based Comment Connection pagination that are flagged.
|
||||
// { storyID, ...connectionParams }
|
||||
await variants(createIndex, {
|
||||
tenantID: 1,
|
||||
storyID: 1,
|
||||
status: 1,
|
||||
"actionCounts.FLAG": 1,
|
||||
});
|
||||
|
||||
// Story + Reply based Comment Connection pagination.
|
||||
// { storyID, ...connectionParams }
|
||||
await variants(createIndex, {
|
||||
tenantID: 1,
|
||||
storyID: 1,
|
||||
parentID: 1,
|
||||
status: 1,
|
||||
});
|
||||
|
||||
// Author based Comment Connection pagination.
|
||||
// { authorID, ...connectionParams }
|
||||
await variants(createIndex, {
|
||||
tenantID: 1,
|
||||
authorID: 1,
|
||||
status: 1,
|
||||
});
|
||||
}
|
||||
|
||||
export type CreateCommentInput = Omit<
|
||||
Comment,
|
||||
| "id"
|
||||
@@ -670,6 +721,7 @@ function applyInputToQuery(
|
||||
input: CommentConnectionInput,
|
||||
query: Query<Comment>
|
||||
) {
|
||||
// NOTE: (wyattjoh) if we ever extend these, ensure that the new order variant is added as an index into the `createCommentConnectionOrderVariants` function.
|
||||
switch (input.orderBy) {
|
||||
case GQLCOMMENT_SORT.CREATED_AT_DESC:
|
||||
query.orderBy({ createdAt: -1 });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { merge } from "lodash";
|
||||
import { FilterQuery } from "./query";
|
||||
|
||||
import { FilterQuery } from "talk-server/models/helpers/query";
|
||||
|
||||
export type Cursor = Date | number | string | null;
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { merge } from "lodash";
|
||||
import { Collection, Cursor } from "mongodb";
|
||||
import { FilterQuery as MongoFilterQuery } from "mongodb";
|
||||
import { Writeable } from "../../common/types";
|
||||
import {
|
||||
Collection,
|
||||
Cursor,
|
||||
FilterQuery as MongoFilterQuery,
|
||||
IndexOptions,
|
||||
} from "mongodb";
|
||||
|
||||
import { Writeable } from "talk-common/types";
|
||||
import logger from "talk-server/logger";
|
||||
|
||||
/**
|
||||
* FilterQuery<T> ensures that given the type T, that the FilterQuery will be a
|
||||
@@ -90,3 +96,69 @@ export default class Query<T> {
|
||||
return cursor;
|
||||
}
|
||||
}
|
||||
|
||||
type IndexType = 1 | -1;
|
||||
|
||||
export type IndexSpecification<T> = {
|
||||
[P in keyof Writeable<Partial<T>>]: IndexType
|
||||
} &
|
||||
Record<string, IndexType>;
|
||||
|
||||
type IndexCreationFunction<T> = (
|
||||
indexSpec: IndexSpecification<T>,
|
||||
indexOptions?: IndexOptions
|
||||
) => Promise<string>;
|
||||
|
||||
export function createIndexFactory<T>(
|
||||
collection: Collection<T>
|
||||
): IndexCreationFunction<T> {
|
||||
const log = logger.child({
|
||||
collectionName: collection.collectionName,
|
||||
});
|
||||
|
||||
return async (
|
||||
indexSpec: IndexSpecification<T>,
|
||||
indexOptions: IndexOptions = {}
|
||||
) => {
|
||||
try {
|
||||
// Try to create the index.
|
||||
const indexName = await collection.createIndex(indexSpec, indexOptions);
|
||||
log.debug({ indexName, indexSpec, indexOptions }, "index was created");
|
||||
|
||||
// Match the interface from the `createIndex` function by returning the
|
||||
// index name.
|
||||
return indexName;
|
||||
} catch (err) {
|
||||
log.error({ err, indexSpec, indexOptions }, "could not create index");
|
||||
|
||||
// Rethrow the error here.
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function createConnectionOrderVariants<T>(
|
||||
variants: Array<IndexSpecification<T>>
|
||||
) {
|
||||
return async (
|
||||
createIndex: IndexCreationFunction<T>,
|
||||
indexSpec: IndexSpecification<T>
|
||||
) => {
|
||||
/**
|
||||
* createIndexVariant will create a variant on the specified `indexSpec` that
|
||||
* will include the new variation.
|
||||
*
|
||||
* @param variantSpec the spec that makes this variant different
|
||||
*/
|
||||
const createIndexVariant = (variantSpec: IndexSpecification<T>) =>
|
||||
createIndex(merge({}, indexSpec, variantSpec));
|
||||
|
||||
// Create a raw index without the variants applied.
|
||||
await createIndex(indexSpec);
|
||||
|
||||
// Create all the variants.
|
||||
for (const variant of variants) {
|
||||
await createIndexVariant(variant);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,15 +1,23 @@
|
||||
import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
|
||||
import { CommentModerationQueueCounts, CommentStatusCounts } from ".";
|
||||
import {
|
||||
CommentModerationCountsPerQueue,
|
||||
CommentModerationQueueCounts,
|
||||
CommentStatusCounts,
|
||||
} from ".";
|
||||
|
||||
export function createEmptyCommentModerationCountsPerQueue(): CommentModerationCountsPerQueue {
|
||||
return {
|
||||
unmoderated: 0,
|
||||
reported: 0,
|
||||
pending: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function createEmptyCommentModerationQueueCounts(): CommentModerationQueueCounts {
|
||||
return {
|
||||
total: 0,
|
||||
queues: {
|
||||
unmoderated: 0,
|
||||
reported: 0,
|
||||
pending: 0,
|
||||
},
|
||||
queues: createEmptyCommentModerationCountsPerQueue(),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,17 +1,18 @@
|
||||
export * from "./empty";
|
||||
export * from "./shared";
|
||||
|
||||
import { identity, isEmpty, pickBy } from "lodash";
|
||||
import { Db } from "mongodb";
|
||||
|
||||
import { identity, isEmpty, pickBy } from "lodash";
|
||||
import { DeepPartial } from "talk-common/types";
|
||||
import { dotize } from "talk-common/utils/dotize";
|
||||
import { GQLCOMMENT_STATUS } from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import logger from "talk-server/logger";
|
||||
import { EncodedCommentActionCounts } from "talk-server/models/action/comment";
|
||||
import { createIndexFactory } from "talk-server/models/helpers/query";
|
||||
import { retrieveStory, Story } from "talk-server/models/story";
|
||||
import { AugmentedRedis } from "talk-server/services/redis";
|
||||
|
||||
import { retrieveStory, Story } from "..";
|
||||
import { createEmptyCommentStatusCounts } from "./empty";
|
||||
import { updateSharedCommentCounts } from "./shared";
|
||||
|
||||
@@ -23,6 +24,13 @@ function collection<T = Story>(db: Db) {
|
||||
return db.collection<Readonly<T>>("stories");
|
||||
}
|
||||
|
||||
export async function createStoryCountIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// { createdAt }
|
||||
await createIndex({ tenantID: 1, createdAt: 1 });
|
||||
}
|
||||
|
||||
// TODO: (wyattjoh) write a test to verify that this set of counts is always in sync with GQLCOMMENT_STATUS.
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,15 +4,16 @@ import ms from "ms";
|
||||
|
||||
import logger from "talk-server/logger";
|
||||
import { EncodedCommentActionCounts } from "talk-server/models/action/comment";
|
||||
import { AugmentedPipeline, AugmentedRedis } from "talk-server/services/redis";
|
||||
|
||||
import { Story } from "talk-server/models/story";
|
||||
import {
|
||||
CommentModerationCountsPerQueue,
|
||||
CommentStatusCounts,
|
||||
StoryCounts,
|
||||
} from ".";
|
||||
import { Story } from "..";
|
||||
} from "talk-server/models/story/counts";
|
||||
import { AugmentedPipeline, AugmentedRedis } from "talk-server/services/redis";
|
||||
|
||||
import {
|
||||
createEmptyCommentModerationCountsPerQueue,
|
||||
createEmptyCommentModerationQueueCounts,
|
||||
createEmptyCommentStatusCounts,
|
||||
} from "./empty";
|
||||
@@ -348,21 +349,28 @@ export async function recalculateSharedCommentCounts(
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* stringObjectToNumber will convert an object that has string keys and string
|
||||
* values into string keys and number values.
|
||||
*/
|
||||
function stringObjectToNumber<T>(
|
||||
input: Record<string, string>,
|
||||
defaultValue: number = 0
|
||||
): T {
|
||||
return Object.entries(input).reduce(
|
||||
(acc, [key, value]) => ({
|
||||
...acc,
|
||||
[key]: parseInt(value, 10) || defaultValue,
|
||||
}),
|
||||
{}
|
||||
) as T;
|
||||
function fillAndConvertStringToNumber<
|
||||
T extends { [P in keyof T]?: string } &
|
||||
{ [P in Exclude<keyof T, keyof U>]?: never },
|
||||
U extends { [P in keyof U]: number }
|
||||
>(input: T, initial: U): U {
|
||||
const result: U = Object.assign({}, initial);
|
||||
for (const key in input) {
|
||||
if (!input.hasOwnProperty(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pull out the value.
|
||||
const value: string | undefined = input[key] as any;
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// I know, not ideal, but...
|
||||
(result as any)[key] = parseInt(value, 10) || 0;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -378,7 +386,7 @@ export async function retrieveSharedActionCommentCounts(
|
||||
mongo: Db,
|
||||
redis: AugmentedRedis,
|
||||
tenantID: string
|
||||
) {
|
||||
): Promise<EncodedCommentActionCounts> {
|
||||
const key = commentCountsActionKey(tenantID);
|
||||
const freshKey = freshenKey(key);
|
||||
|
||||
@@ -395,7 +403,10 @@ export async function retrieveSharedActionCommentCounts(
|
||||
return recalculateSharedActionCommentCounts(mongo, redis, tenantID);
|
||||
}
|
||||
|
||||
return stringObjectToNumber<EncodedCommentActionCounts>(actions);
|
||||
return fillAndConvertStringToNumber(
|
||||
actions,
|
||||
{} as EncodedCommentActionCounts
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -411,13 +422,13 @@ export async function retrieveSharedStatusCommentCounts(
|
||||
mongo: Db,
|
||||
redis: AugmentedRedis,
|
||||
tenantID: string
|
||||
) {
|
||||
): Promise<CommentStatusCounts> {
|
||||
const key = commentCountsStatusKey(tenantID);
|
||||
const freshKey = freshenKey(key);
|
||||
|
||||
// Get the values, and the freshness key.
|
||||
const [[, statuses], [, fresh]]: [
|
||||
[Error | undefined, Record<string, string> | null],
|
||||
[Error | undefined, Record<keyof CommentStatusCounts, string> | null],
|
||||
[Error | undefined, string | null]
|
||||
] = await redis
|
||||
.pipeline()
|
||||
@@ -428,7 +439,10 @@ export async function retrieveSharedStatusCommentCounts(
|
||||
return recalculateSharedStatusCommentCounts(mongo, redis, tenantID);
|
||||
}
|
||||
|
||||
return stringObjectToNumber<CommentStatusCounts>(statuses);
|
||||
return fillAndConvertStringToNumber(
|
||||
statuses,
|
||||
createEmptyCommentStatusCounts()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -473,13 +487,16 @@ export async function retrieveSharedModerationQueueQueuesCounts(
|
||||
mongo: Db,
|
||||
redis: AugmentedRedis,
|
||||
tenantID: string
|
||||
) {
|
||||
): Promise<CommentModerationCountsPerQueue> {
|
||||
const key = commentCountsModerationQueueQueuesKey(tenantID);
|
||||
const freshKey = freshenKey(key);
|
||||
|
||||
// Get the values, and the freshness key.
|
||||
const [[, queues], [, fresh]]: [
|
||||
[Error | undefined, Record<string, string> | null],
|
||||
[
|
||||
Error | undefined,
|
||||
Record<keyof CommentModerationCountsPerQueue, string> | null
|
||||
],
|
||||
[Error | undefined, string | null]
|
||||
] = await redis
|
||||
.pipeline()
|
||||
@@ -493,7 +510,10 @@ export async function retrieveSharedModerationQueueQueuesCounts(
|
||||
|
||||
logger.debug({ tenantID }, "comment moderation counts were cached");
|
||||
|
||||
return stringObjectToNumber<CommentModerationCountsPerQueue>(queues);
|
||||
return fillAndConvertStringToNumber(
|
||||
queues,
|
||||
createEmptyCommentModerationCountsPerQueue()
|
||||
);
|
||||
}
|
||||
|
||||
export async function updateSharedCommentCounts(
|
||||
|
||||
@@ -4,8 +4,10 @@ import uuid from "uuid";
|
||||
import { Omit } from "talk-common/types";
|
||||
import { dotize } from "talk-common/utils/dotize";
|
||||
import { GQLStoryMetadata } from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import { createIndexFactory } from "talk-server/models/helpers/query";
|
||||
import { ModerationSettings } from "talk-server/models/settings";
|
||||
import { TenantResource } from "talk-server/models/tenant";
|
||||
|
||||
import {
|
||||
createEmptyCommentModerationQueueCounts,
|
||||
createEmptyCommentStatusCounts,
|
||||
@@ -60,6 +62,16 @@ export interface Story extends TenantResource {
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export async function createStoryIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
|
||||
|
||||
// UNIQUE { url }
|
||||
await createIndex({ tenantID: 1, url: 1 }, { unique: true });
|
||||
}
|
||||
|
||||
export interface UpsertStoryInput {
|
||||
id?: string;
|
||||
url: string;
|
||||
|
||||
@@ -6,6 +6,7 @@ import uuid from "uuid";
|
||||
import { DeepPartial, Omit, Sub } from "talk-common/types";
|
||||
import { dotize, DotizeOptions } from "talk-common/utils/dotize";
|
||||
import { GQLMODERATION_MODE } from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import { createIndexFactory } from "talk-server/models/helpers/query";
|
||||
import { Settings } from "talk-server/models/settings";
|
||||
|
||||
function collection(db: Db) {
|
||||
@@ -42,6 +43,16 @@ export interface Tenant extends Settings {
|
||||
organizationContactEmail: string;
|
||||
}
|
||||
|
||||
export async function createTenantIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ id: 1 }, { unique: true });
|
||||
|
||||
// UNIQUE { domain }
|
||||
await createIndex({ domain: 1 }, { unique: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* CreateTenantInput is the set of properties that can be set when a given
|
||||
* Tenant is created. The remainder of the properties are set from defaults and
|
||||
|
||||
@@ -4,11 +4,14 @@ import uuid from "uuid";
|
||||
|
||||
import { Omit, Sub } from "talk-common/types";
|
||||
import { GQLUSER_ROLE } from "talk-server/graph/tenant/schema/__generated__/types";
|
||||
import { FilterQuery } from "talk-server/models/query";
|
||||
import {
|
||||
createIndexFactory,
|
||||
FilterQuery,
|
||||
} from "talk-server/models/helpers/query";
|
||||
import { TenantResource } from "talk-server/models/tenant";
|
||||
|
||||
function collection(db: Db) {
|
||||
return db.collection<Readonly<User>>("users");
|
||||
function collection(mongo: Db) {
|
||||
return mongo.collection<Readonly<User>>("users");
|
||||
}
|
||||
|
||||
export interface LocalProfile {
|
||||
@@ -66,6 +69,34 @@ export interface User extends TenantResource {
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export async function createUserIndexes(mongo: Db) {
|
||||
const createIndex = createIndexFactory(collection(mongo));
|
||||
|
||||
// UNIQUE { id }
|
||||
await createIndex({ tenantID: 1, id: 1 }, { unique: true });
|
||||
|
||||
// UNIQUE - PARTIAL { lowercaseUsername }
|
||||
await createIndex(
|
||||
{ tenantID: 1, lowercaseUsername: 1 },
|
||||
{
|
||||
unique: true,
|
||||
partialFilterExpression: { lowercaseUsername: { $exists: true } },
|
||||
}
|
||||
);
|
||||
|
||||
// UNIQUE - PARTIAL { email }
|
||||
await createIndex(
|
||||
{ tenantID: 1, email: 1 },
|
||||
{ unique: true, partialFilterExpression: { email: { $exists: true } } }
|
||||
);
|
||||
|
||||
// UNIQUE { profiles.type, profiles.id }
|
||||
await createIndex(
|
||||
{ tenantID: 1, "profiles.type": 1, "profiles.id": 1 },
|
||||
{ unique: true }
|
||||
);
|
||||
}
|
||||
|
||||
function hashPassword(password: string): Promise<string> {
|
||||
return bcrypt.hash(password, 10);
|
||||
}
|
||||
@@ -171,7 +202,7 @@ const createUpsertUserFilter = (user: Readonly<User>) => {
|
||||
};
|
||||
|
||||
export async function retrieveUser(db: Db, tenantID: string, id: string) {
|
||||
return collection(db).findOne({ id, tenantID });
|
||||
return collection(db).findOne({ tenantID, id });
|
||||
}
|
||||
|
||||
export async function retrieveManyUsers(
|
||||
@@ -194,7 +225,7 @@ export async function retrieveManyUsers(
|
||||
export async function retrieveUserWithProfile(
|
||||
db: Db,
|
||||
tenantID: string,
|
||||
profile: Partial<Profile>
|
||||
profile: Partial<Pick<Profile, "id" | "type">>
|
||||
) {
|
||||
return collection(db).findOne({
|
||||
tenantID,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { Db } from "mongodb";
|
||||
|
||||
import logger from "talk-server/logger";
|
||||
import { createCommentActionIndexes } from "talk-server/models/action/comment";
|
||||
import { createCommentModerationActionIndexes } from "talk-server/models/action/moderation/comment";
|
||||
import { createCommentIndexes } from "talk-server/models/comment";
|
||||
import {
|
||||
createStoryCountIndexes,
|
||||
createStoryIndexes,
|
||||
} from "talk-server/models/story";
|
||||
import { createTenantIndexes } from "talk-server/models/tenant";
|
||||
import { createUserIndexes } from "talk-server/models/user";
|
||||
|
||||
type IndexCreationFunction = (mongo: Db) => Promise<void>;
|
||||
|
||||
const indexes: Array<[string, IndexCreationFunction]> = [
|
||||
["users", createUserIndexes],
|
||||
["tenants", createTenantIndexes],
|
||||
["comments", createCommentIndexes],
|
||||
["stories", createStoryIndexes],
|
||||
["stories", createStoryCountIndexes],
|
||||
["commentActions", createCommentActionIndexes],
|
||||
["commentModerationActions", createCommentModerationActionIndexes],
|
||||
];
|
||||
|
||||
/**
|
||||
* ensureIndexes will ensure that all indexes have been created.
|
||||
*
|
||||
* @param mongo a MongoDB Database Connection
|
||||
*/
|
||||
export async function ensureIndexes(mongo: Db) {
|
||||
logger.debug(
|
||||
{ indexGroupCount: indexes.length },
|
||||
"now ensuring indexes are created"
|
||||
);
|
||||
|
||||
// For each of the index functions, call it.
|
||||
for (const [indexGroup, indexFunction] of indexes) {
|
||||
logger.debug({ indexGroup }, "ensuring indexes are created");
|
||||
await indexFunction(mongo);
|
||||
logger.debug({ indexGroup }, "indexes have been created");
|
||||
}
|
||||
|
||||
logger.debug("all indexes have been created");
|
||||
}
|
||||
@@ -50,6 +50,10 @@ export async function install(
|
||||
);
|
||||
}
|
||||
|
||||
// TODO: (wyattjoh) perform any pending migrations.
|
||||
|
||||
// TODO: (wyattjoh) setup database indexes.
|
||||
|
||||
// Create the Tenant.
|
||||
const tenant = await createTenant(mongo, input);
|
||||
|
||||
|
||||
+5
-4
@@ -20,7 +20,7 @@ async function worker(server: Server) {
|
||||
logger.debug("started server worker");
|
||||
|
||||
// Connect the server to databases.
|
||||
await server.connect();
|
||||
await server.connect({ isWorker: true });
|
||||
|
||||
// Start the server.
|
||||
await server.start(app);
|
||||
@@ -31,6 +31,9 @@ async function worker(server: Server) {
|
||||
|
||||
// master will start the master process.
|
||||
async function master(server: Server) {
|
||||
const workerCount = server.config.get("concurrency");
|
||||
logger.debug({ workerCount }, "spawning workers to handle traffic");
|
||||
|
||||
try {
|
||||
// Connect the server to databases.
|
||||
await server.connect();
|
||||
@@ -60,7 +63,7 @@ async function bootstrap() {
|
||||
);
|
||||
|
||||
// Connect the server to databases.
|
||||
await server.connect();
|
||||
await server.connect({});
|
||||
|
||||
// Process jobs.
|
||||
await server.process();
|
||||
@@ -68,8 +71,6 @@ async function bootstrap() {
|
||||
// Start the server.
|
||||
await server.start(app);
|
||||
} else {
|
||||
logger.debug({ workerCount }, "spawning workers to handle traffic");
|
||||
|
||||
// Launch the server start within throng.
|
||||
throng({
|
||||
workers: workerCount,
|
||||
|
||||
Reference in New Issue
Block a user