Compare commits

...
7 Commits
Author SHA1 Message Date
Vinh ca811a3496 Bring back deprecated events temporarily (#2727)
* fix: bring back deprecated events

* chore: update version number
2019-11-21 20:04:42 +00:00
Wyatt Johnson 10719bd874 [CORL-793] Scheduled Indexes (#2723)
* fix: modified digesting to use index

* fix: added index for deletion

* review: cleaning up

* review: fixed log copy
2019-11-21 17:24:09 +00:00
Wyatt Johnson bb23c80004 feat: generalized rate limiting for graphql (#2709) 2019-11-19 14:20:36 -07:00
Wyatt Johnson 2491445579 feat: improved logging (#2719) 2019-11-19 19:36:46 +00:00
Wyatt Johnson 3642c642d6 fix: fixed migration script bugs (#2718) 2019-11-19 19:18:28 +00:00
Wyatt Johnson fce7b08ca0 chore: version bump (#2717) 2019-11-18 23:27:25 +00:00
Wyatt Johnson aa9dcb4e04 [CORL-750] SSO Migration Script Bug (#2715)
* fix: fixed bug with sso migration script

* feat: refactored sso schema

* fix: resolved issue with data race and migration/install ordering
2019-11-18 23:11:20 +00:00
39 changed files with 613 additions and 337 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@coralproject/talk",
"version": "5.3.0",
"version": "5.3.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@coralproject/talk",
"version": "5.3.0",
"version": "5.3.2",
"author": "The Coral Project",
"homepage": "https://coralproject.net/",
"sideEffects": [
@@ -38,6 +38,8 @@ function createMutationContainer<T extends string, I, R>(
);
private commit = (input: I) => {
// TODO: (cvle) These events are deprecated.
this.props.context.eventEmitter.emit(`mutation.${propName}`, input);
return commit(
this.props.context.relayEnvironment,
input,
@@ -69,6 +69,8 @@ export function useFetch<V, R>(
const context = useCoralContext();
return useCallback<FetchProp<typeof fetch>>(
((variables: V) => {
// TODO: (cvle) These events are deprecated.
context.eventEmitter.emit(`fetch.${fetch.name}`, variables);
return fetch.fetch(context.relayEnvironment, variables, context);
}) as any,
[context]
@@ -93,6 +95,11 @@ export function withFetch<N extends string, V, R>(
public static displayName = wrapDisplayName(BaseComponent, "withFetch");
private fetch = (variables: V) => {
// TODO: (cvle) These events are deprecated.
this.props.context.eventEmitter.emit(
`fetch.${fetch.name}`,
variables
);
return fetch.fetch(
this.props.context.relayEnvironment,
variables,
@@ -71,6 +71,8 @@ export function useMutation<I, R>(
const context = useCoralContext();
return useCallback<MutationProp<typeof mutation>>(
((input: I) => {
// TODO: (cvle) These events are deprecated.
context.eventEmitter.emit(`mutation.${mutation.name}`, input);
return mutation.commit(context.relayEnvironment, input, context);
}) as any,
[context]
@@ -98,6 +100,11 @@ export function withMutation<N extends string, I, R>(
);
private commit = (input: I) => {
// TODO: (cvle) These events are deprecated.
this.props.context.eventEmitter.emit(
`mutation.${mutation.name}`,
input
);
return mutation.commit(
this.props.context.relayEnvironment,
input,
@@ -50,6 +50,8 @@ export function useSubscription<V>(
const context = useCoralContext();
return useCallback<SubscriptionProp<typeof subscription>>(
((variables: V) => {
// TODO: (cvle) These events are deprecated.
context.eventEmitter.emit(`subscription.${subscription.name}`, variables);
return subscription.subscribe(
context.relayEnvironment,
variables,
-6
View File
@@ -53,12 +53,6 @@ export const ALLOWED_USERNAME_CHANGE_FREQUENCY = 14 * 86400;
*/
export const SCHEDULED_DELETION_TIMESPAN_DAYS = 14;
/**
* COMMENT_LIMIT_WINDOW_SECONDS is the number of seconds that a user has to
* wait in-between writing comments.
*/
export const COMMENT_LIMIT_WINDOW_SECONDS = 3;
/**
* DEFAULT_SESSION_LENTTH is the length of time in seconds a session is valid for unless configured in tenant.
*/
+5 -3
View File
@@ -205,6 +205,11 @@ export const installHandler = ({
locale = config.get("default_locale") as LanguageCode;
}
// Execute the pending migrations now, as the schema and types are already
// current for the new tenant being installed now. No point in creating
// a tenant when migrations have not been ran yet.
await migrationManager.executePendingMigrations(mongo, redis, true);
// Install will throw if it can not create a Tenant, or it has already been
// installed.
const tenant = await install(
@@ -248,9 +253,6 @@ export const installHandler = ({
req.coral.now
);
// Execute pending migrations to get everything installed.
await migrationManager.executePendingMigrations(mongo, true);
// Send back the Tenant.
return res.sendStatus(204);
} catch (err) {
@@ -40,6 +40,7 @@ const persistedQueryMiddleware = ({
) {
throw new RawQueryNotAuthorized(
tenant.id,
body && body.query ? body.query : null,
req.user ? req.user.id : null
);
}
+12 -15
View File
@@ -6,6 +6,16 @@ import {
ErrorRequestHandler,
RequestHandler,
} from "coral-server/types/express";
import { Request, Response } from "express";
const extractMetadata = (req: Request, res: Response) => ({
url: req.originalUrl || req.url,
method: req.method,
statusCode: res.statusCode,
host: req.hostname,
userAgent: req.get("User-Agent"),
ip: req.ip,
});
export const accessLogger: RequestHandler = (req, res, next) => {
const startTime = now();
@@ -14,24 +24,11 @@ export const accessLogger: RequestHandler = (req, res, next) => {
// Compute the end time.
const responseTime = Math.round(now() - startTime);
// Get some extra goodies from the request.
const userAgent = req.get("User-Agent");
// Grab the logger.
const log = req.coral ? req.coral.logger : logger;
// Log this out.
log.debug(
{
url: req.originalUrl || req.url,
method: req.method,
statusCode: res.statusCode,
host: req.hostname,
userAgent,
responseTime,
},
"http request"
);
log.debug({ ...extractMetadata(req, res), responseTime }, "http request");
});
next();
@@ -42,7 +39,7 @@ export const errorLogger: ErrorRequestHandler = (err, req, res, next) => {
const log = req.coral ? req.coral.logger : logger;
// Log this out.
log.error({ err }, "http error");
log.error({ ...extractMetadata(req, res), err }, "http error");
next(err);
};
@@ -5,10 +5,7 @@ import { Db } from "mongodb";
import { validate } from "coral-server/app/request/body";
import { IntegrationDisabled, TokenInvalidError } from "coral-server/errors";
import {
RequiredSSOKey,
SSOAuthIntegration,
} from "coral-server/models/settings";
import { SSOAuthIntegration, SSOKey } from "coral-server/models/settings";
import { Tenant } from "coral-server/models/tenant";
import {
retrieveUserWithProfile,
@@ -177,23 +174,15 @@ export function getRelevantSSOKeys(
tokenString: string,
now: Date,
kid?: string
): RequiredSSOKey[] {
): SSOKey[] {
// Collect all the current valid keys.
const keys = integration.keys.filter(k => {
if (!k.secret) {
return false;
}
if (k.deletedAt) {
return false;
}
if (k.deprecateAt && now >= k.deprecateAt) {
if (k.inactiveAt && now >= k.inactiveAt) {
return false;
}
return k;
}) as RequiredSSOKey[];
});
// If there is only one key, that's all we can use!
if (keys.length === 1) {
+3 -2
View File
@@ -749,10 +749,11 @@ export class PersistedQueryNotFound extends CoralError {
}
export class RawQueryNotAuthorized extends CoralError {
constructor(tenantID: string, userID: string | null) {
constructor(tenantID: string, query: string | null, userID: string | null) {
super({
code: ERROR_CODES.RAW_QUERY_NOT_AUTHORIZED,
context: { tenantID, pvt: { userID } },
status: 400,
context: { tenantID, pvt: { userID, query } },
});
}
}
@@ -7,16 +7,18 @@ import {
UserSuspended,
} from "coral-server/errors";
import CommonContext from "coral-server/graph/common/context";
import {
GQLUSER_AUTH_CONDITIONS,
GQLUSER_ROLE,
} from "coral-server/graph/tenant/schema/__generated__/types";
import {
consolidateUserStatus,
consolidateUserSuspensionStatus,
User,
} from "coral-server/models/user";
import { GraphQLResolveInfo, ResponsePath } from "graphql";
import {
GQLUSER_AUTH_CONDITIONS,
GQLUSER_ROLE,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { calculateLocationKey } from "./helpers";
// Replace `memoize.Cache`.
memoize.Cache = WeakMap;
@@ -58,38 +60,6 @@ function calculateAuthConditions(
return conditions.sort();
}
/**
* calculateLocationKey will reduce the resolve information to determine the
* path to where the key that is being accessed.
*
* @param info the info from the graph request
*/
function calculateLocationKey(info: Pick<GraphQLResolveInfo, "path">): string {
// Guard against invalid input.
if (!info || !info.path || !info.path.key) {
return "";
}
// Grab the first part of the path.
const parts: string[] = [info.path.key.toString()];
// Grab the parent previous part of the path.
let prev: ResponsePath | undefined = info.path.prev;
// While there is still a previous part of the path, keep looping to find the
// all the parts.
while (prev && prev.key) {
// Push the key into the front of the array.
parts.unshift(prev.key.toString());
// Change the selection to the previous path element.
prev = prev.prev;
}
// Join it together with a dotted path.
return parts.join(".");
}
const calculateAuthConditionsMemoized = memoize(calculateAuthConditions);
const auth: DirectiveResolverFn<
@@ -0,0 +1,35 @@
import { GraphQLResolveInfo, ResponsePath } from "graphql";
/**
* calculateLocationKey will reduce the resolve information to determine the
* path to where the key that is being accessed.
*
* @param info the info from the graph request
*/
export function calculateLocationKey(
info: Pick<GraphQLResolveInfo, "path" | "operation" | "parentType">
): string {
// Guard against invalid input.
if (!info || !info.path || !info.path.key) {
return "";
}
// Grab the first part of the path.
const parts: string[] = [info.path.key.toString()];
// Grab the parent previous part of the path.
let prev: ResponsePath | undefined = info.path.prev;
// While there is still a previous part of the path, keep looping to find the
// all the parts.
while (prev && prev.key) {
// Push the key into the front of the array.
parts.unshift(prev.key.toString());
// Change the selection to the previous path element.
prev = prev.prev;
}
// Join it together with a dotted path.
return parts.join(".");
}
@@ -0,0 +1,69 @@
import { DirectiveResolverFn } from "graphql-tools";
import { DateTime } from "luxon";
import { RateLimitExceeded } from "coral-server/errors";
import { calculateLocationKey } from "coral-server/graph/common/directives/helpers";
import TenantContext from "../context";
export interface RateDirectiveArgs {
max?: number;
seconds?: number;
key?: string;
}
const rate: DirectiveResolverFn<
Record<string, string | undefined>,
TenantContext
> = async (
next,
src,
{ max = 1, seconds, key: forceResource }: RateDirectiveArgs,
{ user, tenant, now, redis, config },
info
) => {
// If we're in development mode and rate limiters are disabled, then just
// continue anyways now.
if (
config.get("env") === "development" &&
config.get("disable_rate_limiters")
) {
return next();
}
// Check if the rate limiting makes sense.
if (!seconds) {
return next();
}
// Current implementations do not handle anonymous requests.
if (!user) {
// TODO: (wyattjoh) handle anonymous requests
return next();
}
// Compute the resource key for this element.
const resource = forceResource || calculateLocationKey(info);
// TODO: (wyattjoh) depending on `resource`, maybe override (max, seconds)
// Calculate the storage key from the resource and user identifiers.
const key = `${tenant.id}:rl:${user.id}:${info.operation.operation}.${resource}`;
// Perform the rate limiting check.
const [[, tries]] = await redis
.multi()
.incr(key)
.expire(key, seconds)
.exec();
if (tries && tries > max) {
const resetsAt = DateTime.fromJSDate(now)
.plus({ seconds })
.toJSDate();
throw new RateLimitExceeded(key, max, resetsAt, tries);
}
return next();
};
export default rate;
@@ -3,9 +3,8 @@ import * as settings from "coral-server/models/settings";
import { GQLSSOAuthIntegrationTypeResolver } from "coral-server/graph/tenant/schema/__generated__/types";
function getActiveSSOKey(keys: settings.SSOKey[]) {
return keys.find(
key => Boolean(key.secret) && !key.deletedAt && !key.deprecateAt
);
// Any key that has been rotated cannot be the active key.
return keys.find(key => !key.rotatedAt);
}
export const SSOAuthIntegration: GQLSSOAuthIntegrationTypeResolver<
+2 -1
View File
@@ -7,13 +7,14 @@ import {
import { loadSchema } from "coral-common/graphql";
import auth from "coral-server/graph/common/directives/auth";
import constraint from "coral-server/graph/common/directives/constraint";
import rate from "coral-server/graph/tenant/directives/rate";
import resolvers from "coral-server/graph/tenant/resolvers";
export default function getTenantSchema() {
const schema = loadSchema("tenant", resolvers as IResolvers);
// Attach the directive resolvers.
attachDirectiveResolvers(schema, { auth });
attachDirectiveResolvers(schema, { auth, rate });
// Attach the constraint directive.
SchemaDirectiveVisitor.visitSchemaDirectives(schema, {
@@ -60,6 +60,11 @@ arguments to parameters passed in to operations.
"""
directive @constraint(min: Int, max: Int) on ARGUMENT_DEFINITION
"""
rate enforces a rate limit on requests made by the user.
"""
directive @rate(max: Int = 1, seconds: Int!, key: String) on FIELD_DEFINITION
################################################################################
## Custom Scalar Types
################################################################################
@@ -5119,6 +5124,7 @@ type RequestUserCommentsDownloadPayload {
"""
archiveURL: String!
}
##################
## Mutation
##################
@@ -5127,7 +5133,9 @@ type Mutation {
"""
createComment will create a Comment as the current logged in User.
"""
createComment(input: CreateCommentInput!): CreateCommentPayload! @auth
createComment(input: CreateCommentInput!): CreateCommentPayload!
@auth
@rate(seconds: 3, key: "createComment")
"""
createCommentReply will create a Comment as the current logged in User that is
@@ -5135,7 +5143,7 @@ type Mutation {
"""
createCommentReply(
input: CreateCommentReplyInput!
): CreateCommentReplyPayload! @auth
): CreateCommentReplyPayload! @auth @rate(seconds: 3, key: "createComment")
"""
editComment will allow the author of a comment to change the body within the
@@ -5162,7 +5170,7 @@ type Mutation {
"""
createCommentReaction(
input: CreateCommentReactionInput!
): CreateCommentReactionPayload @auth
): CreateCommentReactionPayload @auth @rate(max: 2, seconds: 1)
"""
removeCommentReaction will remove a Reaction authored by the current logged in
@@ -5170,7 +5178,7 @@ type Mutation {
"""
removeCommentReaction(
input: RemoveCommentReactionInput!
): RemoveCommentReactionPayload @auth
): RemoveCommentReactionPayload @auth @rate(max: 2, seconds: 1)
"""
createCommentDontAgree will create a DontAgree authored by the current logged in
@@ -5178,7 +5186,7 @@ type Mutation {
"""
createCommentDontAgree(
input: CreateCommentDontAgreeInput!
): CreateCommentDontAgreePayload @auth
): CreateCommentDontAgreePayload @auth @rate(seconds: 3)
"""
removeCommentDontAgree will remove a DontAgree authored by the current logged in
@@ -5186,7 +5194,7 @@ type Mutation {
"""
removeCommentDontAgree(
input: RemoveCommentDontAgreeInput!
): RemoveCommentDontAgreePayload @auth
): RemoveCommentDontAgreePayload @auth @rate(seconds: 3)
"""
createCommentFlag will create a Flag authored by the current logged in User on
@@ -5194,6 +5202,7 @@ type Mutation {
"""
createCommentFlag(input: CreateCommentFlagInput!): CreateCommentFlagPayload!
@auth
@rate(seconds: 3)
"""
featureComment will mark a given Comment as featured.
@@ -5286,11 +5295,11 @@ type Mutation {
)
"""
updateUsername will set the username on the current User if they have not set one
before. This mutation will fail if the username is already set.
updateUsername will update the users username.
"""
updateUsername(input: UpdateUsernameInput!): UpdateUsernamePayload!
@auth(permit: [SUSPENDED, BANNED, PENDING_DELETION])
@rate(seconds: 10)
"""
setEmail will set the email address on the current User if they have not set
@@ -5311,7 +5320,9 @@ type Mutation {
updatePassword allows the current logged in User to change their password if
they already have one associated with them.
"""
updatePassword(input: UpdatePasswordInput!): UpdatePasswordPayload! @auth
updatePassword(input: UpdatePasswordInput!): UpdatePasswordPayload!
@auth
@rate(seconds: 10)
"""
requestAccountDeletion allows the current logged in User to request to
@@ -5319,7 +5330,7 @@ type Mutation {
"""
requestAccountDeletion(
input: RequestAccountDeletionInput!
): RequestAccountDeletionPayload! @auth
): RequestAccountDeletionPayload! @auth @rate(seconds: 10)
"""
deleteUserAccount will delete the target user now.
@@ -5359,10 +5370,11 @@ type Mutation {
): UpdateUserUsernamePayload! @auth(roles: [ADMIN])
"""
updateEmail allows administrators to update a given User's email address
to the one provided.
updateEmail will update the current users email address.
"""
updateEmail(input: UpdateEmailInput!): UpdateEmailPayload! @auth
updateEmail(input: UpdateEmailInput!): UpdateEmailPayload!
@auth
@rate(seconds: 10)
"""
updateNotificationSettings can be used to update the notification settings for
@@ -237,6 +237,9 @@ export function onOperation(options: OnOperationOptions) {
) {
throw new RawQueryNotAuthorized(
params.context.tenant.id,
message.payload && message.payload.query
? message.payload.query
: null,
params.context.user ? params.context.user.id : null
);
}
+4 -1
View File
@@ -203,7 +203,10 @@ class Server {
// Run migrations if there is already a Tenant installed.
if (await isInstalled(this.tenantCache)) {
await this.migrationManager.executePendingMigrations(this.mongo);
await this.migrationManager.executePendingMigrations(
this.mongo,
this.redis
);
await this.tenantCache.primeAll();
} else {
logger.info("no tenants are installed, skipping running migrations");
+1 -1
View File
@@ -47,7 +47,7 @@ export async function createInvite(
};
// Insert it into the database. This may throw an error.
await collection(mongo).insert(invite);
await collection(mongo).insertOne(invite);
return invite;
}
+9 -12
View File
@@ -1,4 +1,4 @@
import { Omit, RequireProperty } from "coral-common/types";
import { Omit } from "coral-common/types";
import {
GQLAuth,
@@ -46,30 +46,27 @@ export interface SSOKey {
kid: string;
/**
* secret is the actual underlying secret used to verify the tokens with. When
* this is not available, it indicates that the token secret was deleted.
* secret is the actual underlying secret used to verify the tokens with.
*/
secret?: string;
secret: string;
/**
* createdAt is the time that this key was created at.
* createdAt is the date that the key was created at.
*/
createdAt: Date;
/**
* deprecateAt when provided is the time that the token should no longer be
* valid at.
* rotatedAt is the time that the token was rotated out.
*/
deprecateAt?: Date;
rotatedAt?: Date;
/**
* deletedAt is the timestamp that the token was revoked.
* inactiveAt is the date that the token can no longer be used to validate
* tokens.
*/
deletedAt?: Date;
inactiveAt?: Date;
}
export type RequiredSSOKey = RequireProperty<SSOKey, "secret">;
export interface SSOAuthIntegration {
enabled: boolean;
allowRegistration: boolean;
+6 -4
View File
@@ -199,7 +199,7 @@ export async function createTenant(
};
// Insert the Tenant into the database.
await collection(mongo).insert(tenant);
await collection(mongo).insertOne(tenant);
return tenant;
}
@@ -306,18 +306,20 @@ export async function createTenantSSOKey(mongo: Db, id: string, now: Date) {
return result.value || null;
}
export async function deprecateTenantSSOKey(
export async function rotateTenantSSOKey(
mongo: Db,
id: string,
kid: string,
deprecateAt: Date
inactiveAt: Date,
now: Date
) {
// Update the tenant.
const result = await collection(mongo).findOneAndUpdate(
{ id },
{
$set: {
"auth.integrations.sso.keys.$[keys].deprecateAt": deprecateAt,
"auth.integrations.sso.keys.$[keys].inactiveAt": inactiveAt,
"auth.integrations.sso.keys.$[keys].rotatedAt": now,
},
},
{
+10 -2
View File
@@ -420,6 +420,11 @@ export interface User extends TenantResource {
*/
digests: Digest[];
/**
* hasDigests is true when there is digests to send.
*/
hasDigests?: boolean;
/**
* status stores the user status information regarding moderation state.
*/
@@ -2270,6 +2275,9 @@ export async function insertUserNotificationDigests(
$push: {
digests: { $each: digests },
},
$set: {
hasDigests: true,
},
},
{
// False to return the updated document instead of the original
@@ -2308,9 +2316,9 @@ export async function pullUserNotificationDigests(
{
tenantID,
"notifications.digestFrequency": frequency,
digests: { $ne: [] },
hasDigests: true,
},
{ $set: { digests: [] } },
{ $set: { digests: [], hasDigests: false } },
{
// True to return the original document instead of the updated document.
returnOriginal: true,
@@ -49,10 +49,7 @@ import {
import { AugmentedRedis } from "coral-server/services/redis";
import { Request } from "coral-server/types/express";
import {
updateUserLastCommentID,
updateUserLastWroteCommentTimestamp,
} from "../users";
import { updateUserLastCommentID } from "../users";
import { addCommentActions, CreateAction } from "./actions";
import { calculateCounts, calculateCountsDiff } from "./moderation/counts";
import { PhaseResult, processForModeration } from "./pipeline";
@@ -170,10 +167,6 @@ export async function create(
actionCounts = encodeActionCounts(...deDuplicatedActions);
}
// Create the comment action in our rate limiter. This will throw an error if
// there is a rate limit error.
await updateUserLastWroteCommentTimestamp(redis, tenant, author, now);
// Create the comment!
const comment = await createComment(
mongo,
@@ -13,14 +13,12 @@ import { spam } from "./spam";
import { staff } from "./staff";
import { storyClosed } from "./storyClosed";
import { toxic } from "./toxic";
import { userRateLimit } from "./userRateLimit";
import { wordList } from "./wordList";
/**
* The moderation phases to apply for each comment being processed.
*/
export const moderationPhases: IntermediateModerationPhase[] = [
userRateLimit,
commentLength,
storyClosed,
commentingDisabled,
@@ -1,59 +0,0 @@
import { DateTime } from "luxon";
import { COMMENT_LIMIT_WINDOW_SECONDS } from "coral-common/constants";
import { RateLimitExceeded } from "coral-server/errors";
import {
IntermediateModerationPhase,
IntermediatePhaseResult,
ModerationPhaseContext,
} from "coral-server/services/comments/pipeline";
import { retrieveUserLastWroteCommentTimestamp } from "coral-server/services/users";
export const userRateLimit: IntermediateModerationPhase = async ({
action,
author,
redis,
config,
tenant,
now,
}: Pick<
ModerationPhaseContext,
"author" | "redis" | "config" | "now" | "tenant" | "action"
>): Promise<IntermediatePhaseResult | void> => {
// If we're in development mode and rate limiters are disabled, then just
// continue anyways now.
if (
config.get("env") === "development" &&
config.get("disable_rate_limiters")
) {
return;
}
// If this is an edit, we don't need to process this again here.
if (action === "EDIT") {
return;
}
// Check when the last comment was written by this user.
const timestamp = await retrieveUserLastWroteCommentTimestamp(
redis,
tenant,
author
);
if (!timestamp) {
// There is no timestamp written for this user, they are definitely allowed
// to write a comment!
return;
}
// Check to see if this timestamp is still within the limit window. If it is,
// reject the comment.
const nextEditTime = DateTime.fromJSDate(timestamp)
.plus({ seconds: COMMENT_LIMIT_WINDOW_SECONDS })
.toJSDate();
if (nextEditTime > now) {
throw new RateLimitExceeded("createComment", 1, nextEditTime);
}
return;
};
+21 -1
View File
@@ -1,4 +1,5 @@
import fs from "fs-extra";
import { Redis } from "ioredis";
import { Db } from "mongodb";
import path from "path";
import now from "performance-now";
@@ -69,6 +70,15 @@ export default class Manager {
const id = parseInt(matches[1], 10);
const name = matches[2];
// Skip this migration if it was disabled.
if (m.default.disabled) {
logger.warn(
{ migrationID: id, migrationName: name },
"skipping disabled migration"
);
continue;
}
// Create the migration instance.
const migration = new m.default({ id, name, i18n });
@@ -133,7 +143,11 @@ export default class Manager {
return records.length > 0 ? records[records.length - 1] : null;
}
public async executePendingMigrations(mongo: Db, silent = false) {
public async executePendingMigrations(
mongo: Db,
redis: Redis,
silent = false
) {
// Error out if this is ran twice.
if (this.ran) {
if (silent) {
@@ -191,6 +205,7 @@ export default class Manager {
if (migration.up) {
// The migration provides an up method, we should run this per Tenant.
// If no tenants are installed, this will essentially be a no-op.
for await (const tenant of this.tenantCache) {
log = log.child({ tenantID: tenant.id }, true);
@@ -246,5 +261,10 @@ export default class Manager {
},
"finished running pending migrations"
);
for await (const tenant of this.tenantCache) {
// Flush the tenant cache now for each tenant.
await this.tenantCache.delete(redis, tenant.id, tenant.domain);
}
}
}
@@ -23,6 +23,11 @@ abstract class Migration {
public readonly logger: Logger;
public readonly i18n: I18n;
/**
* disabled when true will not run the migration.
*/
public static disabled?: boolean;
constructor({ id, name, i18n }: MigrationOptions) {
this.id = id;
this.name = name;
@@ -7,6 +7,10 @@ import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
export default class extends Migration {
// Remove the following line once the migration is ready, otherwise the
// migration will not be ran!
public static disabled = true;
public async up(mongo: Db, tenantID: string) {
throw new Error("migration not implemented");
}
@@ -20,7 +20,7 @@ export default class extends Migration {
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.matchedCount,
modifiedCount: result.modifiedCount,
},
"added empty moderatorNotes array"
);
@@ -1,7 +1,7 @@
import { Db } from "mongodb";
import { SSOKey } from "coral-server/models/settings";
import { generateSSOKey } from "coral-server/models/tenant";
import { generateSSOKey, Tenant } from "coral-server/models/tenant";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
@@ -9,7 +9,7 @@ import { GQLTime } from "coral-server/graph/tenant/schema/__generated__/types";
import { MigrationError } from "../error";
interface Tenant {
interface OldTenant {
auth: {
integrations: {
sso: {
@@ -20,13 +20,21 @@ interface Tenant {
};
}
function isOldTenant(tenant: Tenant | OldTenant): tenant is OldTenant {
if ((tenant as Tenant).auth.integrations.sso.keys) {
return false;
}
return true;
}
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
// Get the Tenant so we can update it. We don't have to worry about two
// migration operations conflicting here because we will lock one instance
// to perform the operations.
const tenant = await collections
.tenants<Tenant>(mongo)
.tenants<OldTenant | Tenant>(mongo)
.findOne({ id: tenantID });
if (!tenant) {
throw new MigrationError(tenantID, "could not find tenant", "tenants", [
@@ -34,11 +42,17 @@ export default class extends Migration {
]);
}
if (!isOldTenant(tenant)) {
this.log(tenantID).info("tenant already has the new format for the keys");
return;
}
// Store the keys in an array.
const keys: SSOKey[] = [];
// Check to see if a key is set.
const sso = tenant.auth.integrations.sso;
if (sso.key && sso.keyGeneratedAt) {
// Create the new SSOKey based on this data.
const key = generateSSOKey(sso.keyGeneratedAt);
@@ -48,6 +62,13 @@ export default class extends Migration {
// Add this key to the set of keys.
keys.push(key);
} else {
throw new MigrationError(
tenantID,
"expected tenant to have at least one SSO key provided, found none",
"tenants",
[tenantID]
);
}
// Update the tenant with the new sso keys.
@@ -0,0 +1,122 @@
import { DateTime } from "luxon";
import { Db } from "mongodb";
import { SSOKey } from "coral-server/models/settings";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { MigrationError } from "../error";
interface OldSSOKey {
kid: string;
secret?: string;
createdAt: Date;
deprecateAt?: Date;
deletedAt?: Date;
}
function isOldSSOKey(key: SSOKey | OldSSOKey): key is OldSSOKey {
if (!key) {
return true;
}
if ((key as SSOKey).inactiveAt) {
return false;
}
if ((key as SSOKey).rotatedAt) {
return false;
}
if (key.secret === "<deleted>") {
return false;
}
if ((key as OldSSOKey).deprecateAt) {
return true;
}
if ((key as OldSSOKey).deletedAt) {
return true;
}
return false;
}
interface OldTenant {
auth: {
integrations: {
sso: {
keys: OldSSOKey[];
};
};
};
}
export default class extends Migration {
public async down(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
// Transform the keys into the new format.
const keys: OldSSOKey[] = tenant.auth.integrations.sso.keys.map(
(key: OldSSOKey | SSOKey): OldSSOKey =>
!isOldSSOKey(key)
? {
kid: key.kid,
secret: key.secret === "<deleted>" ? undefined : key.secret,
createdAt: key.createdAt,
deprecateAt: key.inactiveAt,
}
: key
);
// Update the key to the new format.
await collections
.tenants<OldTenant>(mongo)
.updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
}
public async test(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
if (tenant.auth.integrations.sso.keys.some(key => isOldSSOKey(key))) {
throw new MigrationError(id, "old sso key was found", "tenants", [id]);
}
}
public async up(mongo: Db, id: string) {
// Get the Tenant that stores the keys in the old format.
const tenant = await collections.tenants<OldTenant>(mongo).findOne({ id });
if (!tenant) {
throw new MigrationError(id, "tenant was not found", "tenants", [id]);
}
// Transform the keys into the new format.
const keys: SSOKey[] = tenant.auth.integrations.sso.keys.map(
(key): SSOKey => ({
kid: key.kid,
secret: key.secret || "<deleted>",
createdAt: key.createdAt,
inactiveAt: key.deprecateAt,
rotatedAt: key.deprecateAt
? DateTime.fromJSDate(key.deprecateAt)
.plus({ month: -1 })
.toJSDate()
: undefined,
})
);
// Update the key to the new format.
await collections
.tenants(mongo)
.updateOne({ id }, { $set: { "auth.integrations.sso.keys": keys } });
}
}
@@ -0,0 +1,46 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndex } from "../indexing";
export default class extends Migration {
public async up(mongo: Db, tenantID: string) {
const result = await collections.users(mongo).updateMany(
{
tenantID,
digests: {
$ne: [],
},
},
{
$set: {
hasDigests: true,
},
}
);
this.log(tenantID).warn(
{
matchedCount: result.matchedCount,
modifiedCount: result.modifiedCount,
},
"added hasDigests flag"
);
}
public async indexes(mongo: Db) {
await createIndex(
collections.users(mongo),
{
tenantID: 1,
"notifications.digestFrequency": 1,
hasDigests: 1,
},
{
partialFilterExpression: { hasDigests: { $eq: true } },
background: true,
}
);
}
}
@@ -0,0 +1,22 @@
import { Db } from "mongodb";
import Migration from "coral-server/services/migrate/migration";
import collections from "coral-server/services/mongodb/collections";
import { createIndex } from "../indexing";
export default class extends Migration {
public async indexes(mongo: Db) {
await createIndex(
collections.users(mongo),
{
tenantID: 1,
scheduledDeletionDate: 1,
},
{
partialFilterExpression: { scheduledDeletionDate: { $exists: true } },
background: true,
}
);
}
}
+27 -22
View File
@@ -9,11 +9,11 @@ export type DeconstructionFn<T> = (tenantID: string, value: T) => Promise<void>;
* automatically invalidate tenants that have been updated.
*/
export class TenantCacheAdapter<T> {
private cache = new Map<string, T>();
private tenantCache: TenantCache;
private readonly cache = new Map<string, T>();
private readonly tenantCache: TenantCache;
private readonly deconstructionFn?: DeconstructionFn<T>;
private unsubscribeFn?: () => void;
private deconstructionFn?: DeconstructionFn<T>;
constructor(
tenantCache: TenantCache,
@@ -26,27 +26,32 @@ export class TenantCacheAdapter<T> {
this.subscribe();
}
private handle = async (tenantID: string) => {
// Get the current set value for the item in the cache.
const value = this.get(tenantID);
// Delete the tenant cache item when the tenant changes.
this.cache.delete(tenantID);
if (this.deconstructionFn) {
// The deconstruction function is set. We will check that the value
// exists, and if it does, we will call the function with the given
// identifier, this allows the caller to attach deconstruction
// components to the tenant being removed. The only side affect to
// note is that by the time that the deconstruction function is
// called, the tenant has already been purged from the cache.
if (typeof value !== "undefined") {
await this.deconstructionFn(tenantID, value);
}
}
};
public subscribe() {
if (this.tenantCache.cachingEnabled && !this.unsubscribeFn) {
this.unsubscribeFn = this.tenantCache.subscribe(async tenant => {
// Get the current set value for the item in the cache.
const value = this.get(tenant.id);
// Delete the tenant cache item when the tenant changes.
this.cache.delete(tenant.id);
if (this.deconstructionFn) {
// The deconstruction function is set. We will check that the value
// exists, and if it does, we will call the function with the given
// identifier, this allows the caller to attach deconstruction
// components to the tenant being removed. The only side affect to
// note is that by the time that the deconstruction function is
// called, the tenant has already been purged from the cache.
if (typeof value !== "undefined") {
await this.deconstructionFn(tenant.id, value);
}
}
});
this.unsubscribeFn = this.tenantCache.subscribe(
({ id }) => this.handle(id),
id => this.handle(id)
);
}
}
+119 -53
View File
@@ -1,8 +1,10 @@
import DataLoader from "dataloader";
import { EventEmitter } from "events";
import { Redis } from "ioredis";
import { Db } from "mongodb";
import uuid from "uuid";
import { Omit } from "coral-common/types";
import { Config } from "coral-server/config";
import logger from "coral-server/logger";
import {
@@ -12,38 +14,57 @@ import {
retrieveManyTenantsByDomain,
Tenant,
} from "coral-server/models/tenant";
import { EventEmitter } from "events";
const TENANT_UPDATE_CHANNEL = "tenant";
const TENANT_CACHE_CHANNEL = "TENANT_CACHE_CHANNEL";
const EMITTER_EVENT_NAME = "update";
enum EVENTS {
UPDATE = "UPDATE",
DELETE = "DELETE",
}
export type SubscribeCallback = (tenant: Tenant) => void;
type UpdateSubscribeCallback = (tenant: Tenant) => void;
type DeleteSubscribeCallback = (tenantID: string, tenantDomain: string) => void;
interface TenantUpdateMessage {
type Message = UpdateMessage | DeleteMessage;
interface DeleteMessage {
event: EVENTS.DELETE;
tenantID: string;
tenantDomain: string;
clientApplicationID: string;
}
interface UpdateMessage {
event: EVENTS.UPDATE;
tenant: Tenant;
clientApplicationID: string;
}
/**
* MessageData is a type that is used to select only the data parts of the
* message.
*/
type MessageData<T extends Message> = Omit<T, "clientApplicationID" | "event">;
// TenantCache provides an interface for retrieving tenant stored in local
// memory rather than grabbing it from the database every single call.
export default class TenantCache {
/**
* tenantsByID reference the tenants that have been cached/retrieved by ID.
*/
private tenantsByID: DataLoader<string, Readonly<Tenant> | null>;
private readonly tenantsByID: DataLoader<string, Readonly<Tenant> | null>;
/**
* tenantsByDomain reference the tenants that have been cached/retrieved by
* Domain.
*/
private tenantsByDomain: DataLoader<string, Readonly<Tenant> | null>;
private readonly tenantsByDomain: DataLoader<string, Readonly<Tenant> | null>;
/**
* tenantCountCache stores all the id's of all the Tenant's that have crossed
* it.
*/
private tenantCountCache = new Set<string>();
private readonly tenantCountCache = new Set<string>();
/**
* primed is true when the cache has already been fully primed.
@@ -55,15 +76,15 @@ export default class TenantCache {
* generated by this application from being handled as external messages
* as we should have already processed it.
*/
private clientApplicationID = uuid.v4();
private readonly clientApplicationID = uuid.v4();
private mongo: Db;
private emitter = new EventEmitter();
private readonly mongo: Db;
private readonly emitter = new EventEmitter();
/**
* cachingEnabled is true when tenant caching has been enabled.
*/
public cachingEnabled: boolean;
public readonly cachingEnabled: boolean;
constructor(mongo: Db, subscriber: Redis, config: Config) {
this.cachingEnabled = !config.get("disable_tenant_caching");
@@ -124,7 +145,7 @@ export default class TenantCache {
subscriber.on("message", this.onMessage);
// Subscribe to tenant notifications.
subscriber.subscribe(TENANT_UPDATE_CHANNEL);
subscriber.subscribe(TENANT_CACHE_CHANNEL);
}
}
@@ -187,10 +208,14 @@ export default class TenantCache {
await this.primeAll();
}
// Copy the tenant count cache to prevent race conditions related to
// clearing during iteration.
const cache = new Set(this.tenantCountCache);
// If the tenant's are primed in the cache, then just use the count cache as
// the iteration source.
if (this.primed) {
for (const tenantID of this.tenantCountCache) {
for (const tenantID of cache) {
const tenant = await this.tenantsByID.load(tenantID);
if (!tenant) {
continue;
@@ -210,23 +235,44 @@ export default class TenantCache {
}
}
private onUpdateMessage({ tenant }: MessageData<UpdateMessage>) {
// Update the tenant cache.
this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
this.tenantCountCache.add(tenant.id);
// Publish the event for the connected listeners.
this.emitter.emit(EVENTS.UPDATE, tenant);
}
private onDeleteMessage({
tenantID,
tenantDomain,
}: MessageData<DeleteMessage>) {
// Delete the tenant in the local cache.
this.tenantsByID.clear(tenantID);
this.tenantsByDomain.clear(tenantDomain);
this.tenantCountCache.delete(tenantID);
// Publish the event for the connected listeners.
this.emitter.emit(EVENTS.DELETE, tenantID, tenantDomain);
}
/**
* onMessage is fired every time the client gets a subscription event.
*/
private onMessage = async (
channel: string,
message: string
): Promise<void> => {
private onMessage = async (channel: string, data: string): Promise<void> => {
// Only do things when the message is for tenant.
if (channel !== TENANT_UPDATE_CHANNEL) {
if (channel !== TENANT_CACHE_CHANNEL) {
return;
}
try {
// Updated tenant come from the messages.
const { tenant, clientApplicationID }: TenantUpdateMessage = JSON.parse(
message
);
// Parse the message (which is JSON).
const message: Message = JSON.parse(data);
// Extract some known parameters.
const { clientApplicationID } = message;
// Check to see if this was the update issued by this instance.
if (clientApplicationID === this.clientApplicationID) {
@@ -234,19 +280,23 @@ export default class TenantCache {
return;
}
logger.debug({ tenantID: tenant.id }, "received updated tenant");
const log = logger.child({ eventName: message.event }, true);
log.debug("received tenant message");
// Update the tenant cache.
this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
this.tenantCountCache.add(tenant.id);
// Publish the event for the connected listeners.
this.emitter.emit(EMITTER_EVENT_NAME, tenant);
// Send the message to the correct handler.
switch (message.event) {
case EVENTS.UPDATE:
return this.onUpdateMessage(message);
case EVENTS.DELETE:
return this.onDeleteMessage(message);
default:
log.warn("received unknown event");
return;
}
} catch (err) {
logger.error(
{ err },
"an error occurred while trying to parse/prime the tenant/tenant cache"
"an error occurred while trying to handle a message"
);
}
};
@@ -265,17 +315,34 @@ export default class TenantCache {
* This allows you to subscribe to new Tenant updates. This will also return
* a function that when called, unsubscribes you from updates.
*
* @param callback the function to be called when there is an updated Tenant.
* @param updateCallback the function to be called when there is an updated Tenant.
* @param deleteCallback the function to be called when a tenant needs to be purged
*/
public subscribe(callback: SubscribeCallback) {
this.emitter.on(EMITTER_EVENT_NAME, callback);
public subscribe(
updateCallback: UpdateSubscribeCallback,
deleteCallback: DeleteSubscribeCallback
) {
this.emitter.on(EVENTS.UPDATE, updateCallback);
this.emitter.on(EVENTS.DELETE, deleteCallback);
// Return the unsubscribe function.
return () => {
this.emitter.removeListener(EMITTER_EVENT_NAME, callback);
this.emitter.removeListener(EVENTS.UPDATE, updateCallback);
this.emitter.removeListener(EVENTS.DELETE, deleteCallback);
};
}
private async publish(tenantID: string, conn: Redis, message: Message) {
const subscribers = await conn.publish(
TENANT_CACHE_CHANNEL,
JSON.stringify(message)
);
logger.debug(
{ tenantID, subscribers, eventName: message.event },
"updated tenant in cache"
);
}
/**
* update will update the value for Tenant in the local cache and publish
* a change notification that will be used to keep the other nodes in sync.
@@ -284,28 +351,27 @@ export default class TenantCache {
* @param tenant the updated Tenant object
*/
public async update(conn: Redis, tenant: Tenant): Promise<void> {
// Update the tenant in the local cache.
this.tenantsByID.clear(tenant.id).prime(tenant.id, tenant);
this.tenantsByDomain.clear(tenant.domain).prime(tenant.domain, tenant);
this.tenantCountCache.add(tenant.id);
// Process the tenant update on this node.
this.onUpdateMessage({ tenant });
// Notify the other nodes about the tenant change.
const message: TenantUpdateMessage = {
await this.publish(tenant.id, conn, {
event: EVENTS.UPDATE,
tenant,
clientApplicationID: this.clientApplicationID,
};
});
}
const subscribers = await conn.publish(
TENANT_UPDATE_CHANNEL,
JSON.stringify(message)
);
public async delete(conn: Redis, tenantID: string, tenantDomain: string) {
// Process the tenant update on this node.
this.onDeleteMessage({ tenantID, tenantDomain });
logger.debug(
{ tenantID: tenant.id, subscribers },
"updated tenant in cache"
);
// Publish the event for the connected listeners.
this.emitter.emit(EMITTER_EVENT_NAME, tenant);
// Notify the other nodes about the tenant change.
await this.publish(tenantID, conn, {
event: EVENTS.DELETE,
tenantID,
tenantDomain,
clientApplicationID: this.clientApplicationID,
});
}
}
+3 -3
View File
@@ -12,7 +12,7 @@ import {
createTenant,
CreateTenantInput,
createTenantSSOKey,
deprecateTenantSSOKey,
rotateTenantSSOKey,
Tenant,
updateTenant,
} from "coral-server/models/tenant";
@@ -150,7 +150,7 @@ export async function regenerateSSOKey(
if (tenant.auth.integrations.sso.keys.length > 0) {
// Get the old keys that are not deprecated.
const keysToDeprecate = tenant.auth.integrations.sso.keys.filter(key => {
return !key.deletedAt && !key.deprecateAt && key.secret;
return !key.rotatedAt;
});
// Check to see if there are keys to deprecate.
@@ -164,7 +164,7 @@ export async function regenerateSSOKey(
// Deprecate all the keys that are associated on the tenant that haven't
// been done.
for (const key of keysToDeprecate) {
await deprecateTenantSSOKey(mongo, tenant.id, key.kid, deprecateAt);
await rotateTenantSSOKey(mongo, tenant.id, key.kid, deprecateAt, now);
}
}
}
-68
View File
@@ -3,7 +3,6 @@ import { Db } from "mongodb";
import {
ALLOWED_USERNAME_CHANGE_FREQUENCY,
COMMENT_LIMIT_WINDOW_SECONDS,
COMMENT_REPEAT_POST_TIMESPAN,
DOWNLOAD_LIMIT_TIMEFRAME,
} from "coral-common/constants";
@@ -17,7 +16,6 @@ import {
LocalProfileAlreadySetError,
LocalProfileNotSetError,
PasswordIncorrect,
RateLimitExceeded,
TokenNotFoundError,
UserAlreadyBannedError,
UserAlreadyPremoderated,
@@ -1185,13 +1183,6 @@ export async function updateNotificationSettings(
return updateUserNotificationSettings(mongo, tenant.id, user.id, settings);
}
function userLastWroteCommentTimestampKey(
tenant: Pick<Tenant, "id">,
user: Pick<User, "id">
) {
return `${tenant.id}:lastCommentTimestamp:${user.id}`;
}
function userLastCommentIDKey(
tenant: Pick<Tenant, "id">,
user: Pick<User, "id">
@@ -1199,65 +1190,6 @@ function userLastCommentIDKey(
return `${tenant.id}:lastCommentID:${user.id}`;
}
/**
* retrieveUserLastWroteCommentTimestamp will return the timestamp (if set) that
* the user last wrote a comment on. This will return null if the comment was
* written more than COMMENT_LIMIT_WINDOW_SECONDS seconds ago.
*
* @param redis the Redis instance that Coral interacts with
* @param tenant the Tenant to operate on
* @param user the User that we're looking up the limit for
*/
export async function retrieveUserLastWroteCommentTimestamp(
redis: AugmentedRedis,
tenant: Tenant,
user: User
): Promise<Date | null> {
// Try to get the timestamp for the author.
const timestamp: string | null = await redis.get(
userLastWroteCommentTimestampKey(tenant, user)
);
if (!timestamp) {
return null;
}
return DateTime.fromISO(timestamp).toJSDate();
}
/**
* updateUserLastWroteCommentTimestamp will update the last time that the user
* wrote a comment, and will throw an error if the rate limit was exceeded. If
* this throws an error, it means that the user has written a comment within
* COMMENT_LIMIT_WINDOW_SECONDS seconds, and should be prevented from writing
* another comment.
*
* @param redis the Redis instance that Coral interacts with
* @param tenant the Tenant to operate on
* @param user the User that we're setting the limit for
* @param when the date that the user wrote the comment
*/
export async function updateUserLastWroteCommentTimestamp(
redis: AugmentedRedis,
tenant: Tenant,
user: User,
when: Date
) {
const key = userLastWroteCommentTimestampKey(tenant, user);
// Try to set the last wrote comment timestamp.
const [[, set]] = await redis
.multi()
.setnx(key, when.toISOString())
.expire(key, COMMENT_LIMIT_WINDOW_SECONDS)
.exec();
if (!set) {
const resetsAt = DateTime.fromJSDate(when)
.plus({ seconds: COMMENT_LIMIT_WINDOW_SECONDS })
.toJSDate();
throw new RateLimitExceeded("createComment", 1, resetsAt);
}
}
/**
* updateUserLastCommentID will update the id of the users most recent comment.
*