[CORL-166] Live Updates on Mod Queues (#2368)

* feat: client implementation of subscriptions and modqueue live counts

* fix: unit tests

* feat: live status update in moderation

* feat: live update of new comments in moderation

* chore: View New instead of View More

* feat: fade in transition for new comments

* chore: turn websocket proxy back on

* feat: initial server impl

* fix: make it work :-)

* fix: add box shadow

* chore: make test subscriptions only support 1 top level field following the spec

* fix: linting

* feat: support clientID

* fix: linting

* feat: support commentStatusUpdated subscription

* fix: disabled styles for approve and reject button

* feat: show moderated by system and update flags

* feat: support metrics recording on websocket connections

* fix: handle when same comment enters but leaves again
This commit is contained in:
Vinh
2019-06-21 17:01:07 +00:00
committed by Wyatt Johnson
parent 0e247ba383
commit 413f3e2f1e
111 changed files with 8230 additions and 5017 deletions
@@ -10,7 +10,7 @@ import {
} from "coral-server/errors";
import { GQLUSER_ROLE } from "coral-server/graph/tenant/schema/__generated__/types";
import { retrieveUser, User } from "coral-server/models/user";
import { decodeJWT, extractJWTFromRequest } from "coral-server/services/jwt";
import { decodeJWT, extractTokenFromRequest } from "coral-server/services/jwt";
import {
confirmEmail,
sendConfirmationEmail,
@@ -165,7 +165,7 @@ export const confirmCheckHandler = ({
// TODO: evaluate verifying if the Tenant allows verifications to short circuit.
// Grab the token from the request.
const tokenString = extractJWTFromRequest(req, true);
const tokenString = extractTokenFromRequest(req, true);
if (!tokenString) {
return res.sendStatus(400);
}
@@ -225,7 +225,7 @@ export const confirmHandler = ({
const tenant = coral.tenant!;
// Grab the token from the request.
const tokenString = extractJWTFromRequest(req, true);
const tokenString = extractTokenFromRequest(req, true);
if (!tokenString) {
return res.sendStatus(400);
}
@@ -5,7 +5,7 @@ import { validate } from "coral-server/app/request/body";
import { RequestLimiter } from "coral-server/app/request/limiter";
import { IntegrationDisabled } from "coral-server/errors";
import { retrieveUserWithProfile } from "coral-server/models/user";
import { decodeJWT, extractJWTFromRequest } from "coral-server/services/jwt";
import { decodeJWT, extractTokenFromRequest } from "coral-server/services/jwt";
import {
generateResetURL,
resetPassword,
@@ -182,7 +182,7 @@ export const forgotResetHandler = ({
);
// Grab the token from the request.
const tokenString = extractJWTFromRequest(req, true);
const tokenString = extractTokenFromRequest(req, true);
if (!tokenString) {
return res.sendStatus(400);
}
@@ -248,7 +248,7 @@ export const forgotCheckHandler = ({
}
// Grab the token from the request.
const tokenString = extractJWTFromRequest(req, true);
const tokenString = extractTokenFromRequest(req, true);
if (!tokenString) {
return res.sendStatus(400);
}
+48 -24
View File
@@ -1,48 +1,56 @@
import { CLIENT_ID_HEADER } from "coral-common/constants";
import { AppOptions } from "coral-server/app";
import {
graphqlBatchMiddleware,
graphqlMiddleware,
} from "coral-server/app/middleware/graphql";
import TenantContext from "coral-server/graph/tenant/context";
import TenantContext, {
TenantContextOptions,
} from "coral-server/graph/tenant/context";
import { Request, RequestHandler } from "coral-server/types/express";
export type GraphMiddlewareOptions = Pick<
AppOptions,
| "schema"
| "config"
| "i18n"
| "mailerQueue"
| "mongo"
| "redis"
| "mailerQueue"
| "schema"
| "scraperQueue"
| "signingConfig"
| "i18n"
| "pubsub"
| "tenantCache"
| "metrics"
>;
export const graphQLHandler = ({
schema,
config,
metrics,
...options
}: GraphMiddlewareOptions): RequestHandler =>
graphqlBatchMiddleware(
graphqlMiddleware(config, async (req: Request) => {
if (!req.coral) {
throw new Error("coral was not set");
}
graphqlMiddleware(
config,
async (req: Request) => {
if (!req.coral) {
throw new Error("coral was not set");
}
// Pull out some useful properties from Coral.
const { id, now, tenant, cache, logger } = req.coral;
// Pull out some useful properties from Coral.
const { id, now, tenant, cache, logger } = req.coral;
if (!cache) {
throw new Error("cache was not set");
}
if (!cache) {
throw new Error("cache was not set");
}
if (!tenant) {
throw new Error("tenant was not set");
}
if (!tenant) {
throw new Error("tenant was not set");
}
return {
schema,
context: new TenantContext({
// Create some new options to store the tenant context details inside.
const opts: TenantContextOptions = {
...options,
id,
now,
@@ -50,9 +58,25 @@ export const graphQLHandler = ({
config,
tenant,
logger,
user: req.user,
tenantCache: cache.tenant,
}),
};
})
};
// Add the user if there is one.
if (req.user) {
opts.user = req.user;
}
// Add the clientID if there is one on the request.
const clientID = req.get(CLIENT_ID_HEADER);
if (clientID) {
// TODO: (wyattjoh) validate length
opts.clientID = clientID;
}
return {
schema,
context: new TenantContext(opts),
};
},
metrics
)
);
+21
View File
@@ -0,0 +1,21 @@
import { IncomingMessage } from "http";
/**
* Duplicates the functionality from expressjs:
*
* https://github.com/expressjs/express/blob/b8e50568af9c73ef1ade434e92c60d389868361d/lib/request.js#L416-L450
*
* @param req incoming request
*/
export function getHostname(req: IncomingMessage) {
const host = req.headers["x-forwarded-host"] || req.headers.host;
if (!host || Array.isArray(host)) {
return null;
}
// IPv6 literal support
const offset = host[0] === "[" ? host.indexOf("]") + 1 : 0;
const index = host.indexOf(":", offset);
return index !== -1 ? host.substring(0, index) : host;
}
+8 -3
View File
@@ -2,6 +2,7 @@ import cons from "consolidate";
import cors from "cors";
import { Express } from "express";
import { GraphQLSchema } from "graphql";
import { RedisPubSub } from "graphql-redis-subscriptions";
import http from "http";
import { Db } from "mongodb";
import nunjucks from "nunjucks";
@@ -16,6 +17,7 @@ import { MailerQueue } from "coral-server/queue/tasks/mailer";
import { ScraperQueue } from "coral-server/queue/tasks/scraper";
import { I18n } from "coral-server/services/i18n";
import { JWTSigningConfig } from "coral-server/services/jwt";
import { Metrics } from "coral-server/services/metrics";
import { AugmentedRedis } from "coral-server/services/redis";
import TenantCache from "coral-server/services/tenant/cache";
@@ -35,8 +37,9 @@ export interface AppOptions {
schema: GraphQLSchema;
signingConfig: JWTSigningConfig;
tenantCache: TenantCache;
metrics: boolean;
disableClientRoutes: boolean;
metrics?: Metrics;
pubsub: RedisPubSub;
}
/**
@@ -52,8 +55,10 @@ export async function createApp(options: AppOptions): Promise<Express> {
// Logging
parent.use(accessLogger);
// Capturing metrics.
parent.use(metricsRecorder());
if (options.metrics) {
// Capturing metrics.
parent.use(metricsRecorder(options.metrics));
}
// Create some services for the router.
const passport = createPassport(options);
+17 -26
View File
@@ -1,7 +1,6 @@
import { GraphQLOptions } from "apollo-server-express";
import { GraphQLExtension, GraphQLOptions } from "apollo-server-express";
import { Handler } from "express";
import { FieldDefinitionNode, GraphQLError, ValidationContext } from "graphql";
import { Counter, Histogram } from "prom-client";
// TODO: when https://github.com/apollographql/apollo-server/pull/1907 is merged, update this import path
import {
@@ -16,6 +15,7 @@ import {
LoggerExtension,
MetricsExtension,
} from "coral-server/graph/common/extensions";
import { Metrics } from "coral-server/services/metrics";
export * from "./batch";
@@ -42,37 +42,28 @@ const NoIntrospection = (context: ValidationContext) => ({
*/
export const graphqlMiddleware = (
config: Config,
requestOptions: ExpressGraphQLOptionsFunction
requestOptions: ExpressGraphQLOptionsFunction,
metrics?: Metrics
): Handler => {
// Configure the metrics handlers.
const executedGraphQueriesTotalCounter = new Counter({
name: "coral_executed_graph_queries_total",
help: "number of GraphQL queries executed",
labelNames: ["operation_type", "operation_name"],
});
const extensions: Array<() => GraphQLExtension> = [
() => new ErrorWrappingExtension(),
() => new LoggerExtension(),
];
const graphQLExecutionTimingsHistogram = new Histogram({
name: "coral_executed_graph_queries_timings",
help: "timings for execution times of GraphQL operations",
buckets: [0.1, 5, 15, 50, 100, 500],
labelNames: ["operation_type", "operation_name"],
});
// Add the metrics extension if provided.
if (metrics) {
extensions.push(
() =>
// Pass the metrics to the extension so it can increment.
new MetricsExtension(metrics)
);
}
// Create a new baseOptions that will be merged into the new options.
const baseOptions: Omit<GraphQLOptions, "schema"> = {
// Disable the debug mode, as we already add in our logging function.
debug: false,
// Include extensions.
extensions: [
() => new ErrorWrappingExtension(),
() => new LoggerExtension(),
() =>
// Pass the metrics to the extension so it can increment.
new MetricsExtension({
executedGraphQueriesTotalCounter,
graphQLExecutionTimingsHistogram,
}),
],
extensions,
};
if (config.get("env") === "production" && !config.get("enable_graphiql")) {
+7 -14
View File
@@ -1,22 +1,15 @@
import { RequestHandler } from "express";
import onFinished from "on-finished";
import now from "performance-now";
import { Counter, Histogram } from "prom-client";
export const metricsRecorder = (): RequestHandler => {
const httpRequestsTotal = new Counter({
name: "http_requests_total",
help: "Total number of HTTP requests made.",
labelNames: ["code", "method"],
});
import { Metrics } from "coral-server/services/metrics";
import { RequestHandler } from "coral-server/types/express";
const httpRequestDurationMilliseconds = new Histogram({
name: "http_request_duration_milliseconds",
help: "Histogram of latencies for HTTP requests.",
buckets: [0.1, 5, 15, 50, 100, 500],
labelNames: ["method", "handler"],
});
export type MetricsRecorderOptions = Metrics;
export const metricsRecorder = ({
httpRequestsTotal,
httpRequestDurationMilliseconds,
}: Metrics): RequestHandler => {
return (req, res, next) => {
const startTime = now();
@@ -14,7 +14,7 @@ import { validate } from "coral-server/app/request/body";
import { AuthenticationError } from "coral-server/errors";
import { User } from "coral-server/models/user";
import {
extractJWTFromRequest,
extractTokenFromRequest,
JWTSigningConfig,
revokeJWT,
signTokenString,
@@ -68,7 +68,7 @@ const LogoutTokenSchema = Joi.object().keys({
export async function handleLogout(redis: Redis, req: Request, res: Response) {
// Extract the token from the request.
const token = extractJWTFromRequest(req);
const token = extractTokenFromRequest(req);
if (!token) {
// TODO: (wyattjoh) return a better error.
throw new Error("logout requires a token on the request, none was found");
@@ -5,7 +5,7 @@ import { AppOptions } from "coral-server/app";
import { TenantNotFoundError, TokenInvalidError } from "coral-server/errors";
import { Tenant } from "coral-server/models/tenant";
import { User } from "coral-server/models/user";
import { extractJWTFromRequest } from "coral-server/services/jwt";
import { extractTokenFromRequest } from "coral-server/services/jwt";
import { Request } from "coral-server/types/express";
import { JWTToken, JWTVerifier } from "./verifiers/jwt";
@@ -20,7 +20,7 @@ export type JWTStrategyOptions = Pick<
/**
* Token is the various forms of the Token that can be verified.
*/
type Token = OIDCIDToken | SSOToken | JWTToken | object | string | null;
export type Token = OIDCIDToken | SSOToken | JWTToken | object | string | null;
/**
* Verifier allows different implementations to offer ways to verify a given
@@ -44,6 +44,41 @@ export interface Verifier<T = Token> {
supports: (token: T | object, tenant: Tenant) => token is T;
}
export function createVerifiers(
options: JWTStrategyOptions
): Array<Verifier<Token>> {
return [
new OIDCVerifier(options),
new SSOVerifier(options),
new JWTVerifier(options),
];
}
export function verifyAndRetrieveUser(
verifiers: Array<Verifier<Token>>,
tenant: Tenant,
tokenString: string,
now = new Date()
) {
const token: Token = jwt.decode(tokenString);
if (!token || typeof token === "string") {
throw new TokenInvalidError(tokenString, "token could not be decoded");
}
// Try to verify the token.
for (const verifier of verifiers) {
if (verifier.supports(token, tenant)) {
return verifier.verify(tokenString, token, tenant, now);
}
}
// No verifier could be found.
throw new TokenInvalidError(
tokenString,
"no suitable jwt verifier could be found"
);
}
export class JWTStrategy extends Strategy {
public name = "jwt";
@@ -52,36 +87,12 @@ export class JWTStrategy extends Strategy {
constructor(options: JWTStrategyOptions) {
super();
this.verifiers = [
new OIDCVerifier(options),
new SSOVerifier(options),
new JWTVerifier(options),
];
}
private async verify(tokenString: string, tenant: Tenant, now = new Date()) {
const token: Token = jwt.decode(tokenString);
if (!token || typeof token === "string") {
throw new TokenInvalidError(tokenString, "token could not be decoded");
}
// Try to verify the token.
for (const verifier of this.verifiers) {
if (verifier.supports(token, tenant)) {
return verifier.verify(tokenString, token, tenant, now);
}
}
// No verifier could be found.
throw new TokenInvalidError(
tokenString,
"no suitable jwt verifier could be found"
);
this.verifiers = createVerifiers(options);
}
public async authenticate(req: Request) {
// Get the token from the request.
const token = extractJWTFromRequest(req);
const token = extractTokenFromRequest(req);
if (!token) {
// There was no token on the request, so don't bother actually checking
// anything further.
@@ -94,7 +105,12 @@ export class JWTStrategy extends Strategy {
}
try {
const user = await this.verify(token, tenant, now);
const user = await verifyAndRetrieveUser(
this.verifiers,
tenant,
token,
now
);
if (!user) {
return this.pass();
}
+7 -1
View File
@@ -69,5 +69,11 @@ function attachGraphiQL(router: Router, app: AppOptions) {
}
// GraphiQL
router.get("/graphiql", playground({ endpoint: "/api/graphql" }));
router.get(
"/graphiql",
playground({
endpoint: "/api/graphql",
subscriptionEndpoint: "/api/graphql/live",
})
);
}
+2 -2
View File
@@ -406,10 +406,10 @@ export class StoryNotFoundError extends CoralError {
}
export class CommentNotFoundError extends CoralError {
constructor(commentID: string) {
constructor(commentID: string, commentRevisionID?: string) {
super({
code: ERROR_CODES.COMMENT_NOT_FOUND,
context: { pvt: { commentID } },
context: { pvt: { commentID, commentRevisionID } },
});
}
}
+19
View File
@@ -1,3 +1,4 @@
import { Db } from "mongodb";
import uuid from "uuid";
import { LanguageCode } from "coral-common/helpers/i18n/locales";
@@ -5,7 +6,9 @@ import { Config } from "coral-server/config";
import logger, { Logger } from "coral-server/logger";
import { User } from "coral-server/models/user";
import { I18n } from "coral-server/services/i18n";
import { AugmentedRedis } from "coral-server/services/redis";
import { Request } from "coral-server/types/express";
import { RedisPubSub } from "graphql-redis-subscriptions";
export interface CommonContextOptions {
id?: string;
@@ -14,8 +17,12 @@ export interface CommonContextOptions {
req?: Request;
logger?: Logger;
lang?: LanguageCode;
disableCaching?: boolean;
config: Config;
i18n: I18n;
pubsub: RedisPubSub;
mongo: Db;
redis: AugmentedRedis;
}
export default class CommonContext {
@@ -27,6 +34,10 @@ export default class CommonContext {
public readonly lang: LanguageCode;
public readonly now: Date;
public readonly logger: Logger;
public readonly pubsub: RedisPubSub;
public readonly mongo: Db;
public readonly redis: AugmentedRedis;
public readonly disableCaching: boolean;
constructor({
id = uuid.v1(),
@@ -37,6 +48,10 @@ export default class CommonContext {
config,
i18n,
lang = i18n.getDefaultLang(),
pubsub,
mongo,
redis,
disableCaching = false,
}: CommonContextOptions) {
this.id = id;
this.logger = log.child({
@@ -49,5 +64,9 @@ export default class CommonContext {
this.config = config;
this.i18n = i18n;
this.lang = lang;
this.pubsub = pubsub;
this.mongo = mongo;
this.redis = redis;
this.disableCaching = disableCaching;
}
}
@@ -1,4 +1,4 @@
import { ExecutionArgs, GraphQLError } from "graphql";
import { DocumentNode, ExecutionArgs, GraphQLError } from "graphql";
import {
EndHandler,
GraphQLExtension,
@@ -13,6 +13,20 @@ export function logError(ctx: CommonContext, err: GraphQLError) {
ctx.logger.error({ err }, "graphql query error");
}
export function logQuery(
ctx: CommonContext,
document: DocumentNode,
responseTime?: number
) {
ctx.logger.debug(
{
responseTime,
...getOperationMetadata(document),
},
"graphql query"
);
}
export class LoggerExtension implements GraphQLExtension<CommonContext> {
public executionDidStart(o: {
executionArgs: ExecutionArgs;
@@ -27,12 +41,10 @@ export class LoggerExtension implements GraphQLExtension<CommonContext> {
const responseTime = Math.round(now() - startTime);
// Log out the details of the request.
o.executionArgs.contextValue.logger.debug(
{
responseTime,
...getOperationMetadata(o.executionArgs.document),
},
"graphql query"
logQuery(
o.executionArgs.contextValue,
o.executionArgs.document,
responseTime
);
};
}
@@ -1,22 +1,13 @@
import CommonContext from "coral-server/graph/common/context";
import { Metrics } from "coral-server/services/metrics";
import { ExecutionArgs } from "graphql";
import { EndHandler, GraphQLExtension } from "graphql-extensions";
import now from "performance-now";
import { Counter, Histogram } from "prom-client";
import CommonContext from "coral-server/graph/common/context";
import { getOperationMetadata } from "./helpers";
export interface MetricsExtensionOptions {
executedGraphQueriesTotalCounter: Counter;
graphQLExecutionTimingsHistogram: Histogram;
}
export class MetricsExtension implements GraphQLExtension<CommonContext> {
private options: MetricsExtensionOptions;
constructor(options: MetricsExtensionOptions) {
this.options = options;
}
constructor(private metrics: Metrics) {}
public executionDidStart(o: {
executionArgs: ExecutionArgs;
@@ -37,11 +28,11 @@ export class MetricsExtension implements GraphQLExtension<CommonContext> {
if (operation && operationName) {
// Increment the graph query value, tagging with the name of the query.
this.options.executedGraphQueriesTotalCounter
this.metrics.executedGraphQueriesTotalCounter
.labels(operation, operationName)
.inc();
this.options.graphQLExecutionTimingsHistogram
this.metrics.graphQLExecutionTimingsHistogram
.labels(operation, operationName)
.observe(responseTime);
}
@@ -0,0 +1,12 @@
import { RedisPubSub } from "graphql-redis-subscriptions";
import { Redis } from "ioredis";
export function createPubSubClient(
publisher: Redis,
subscriber: Redis
): RedisPubSub {
return new RedisPubSub({
publisher,
subscriber,
});
}
+16 -12
View File
@@ -1,52 +1,56 @@
import { Db } from "mongodb";
import CommonContext, {
CommonContextOptions,
} from "coral-server/graph/common/context";
import {
createPublisher,
Publisher,
} from "coral-server/graph/tenant/subscriptions/publisher";
import { Tenant } from "coral-server/models/tenant";
import { User } from "coral-server/models/user";
import { MailerQueue } from "coral-server/queue/tasks/mailer";
import { ScraperQueue } from "coral-server/queue/tasks/scraper";
import { JWTSigningConfig } from "coral-server/services/jwt";
import { AugmentedRedis } from "coral-server/services/redis";
import TenantCache from "coral-server/services/tenant/cache";
import loaders from "./loaders";
import mutators from "./mutators";
export interface TenantContextOptions extends CommonContextOptions {
mongo: Db;
redis: AugmentedRedis;
tenant: Tenant;
tenantCache: TenantCache;
mailerQueue: MailerQueue;
scraperQueue: ScraperQueue;
signingConfig?: JWTSigningConfig;
clientID?: string;
}
export default class TenantContext extends CommonContext {
public readonly tenant: Tenant;
public readonly tenantCache: TenantCache;
public readonly mongo: Db;
public readonly redis: AugmentedRedis;
public readonly mailerQueue: MailerQueue;
public readonly scraperQueue: ScraperQueue;
public readonly loaders: ReturnType<typeof loaders>;
public readonly mutators: ReturnType<typeof mutators>;
public readonly publisher: Publisher;
public readonly user?: User;
public readonly signingConfig?: JWTSigningConfig;
public readonly clientID?: string;
public readonly loaders: ReturnType<typeof loaders>;
public readonly mutators: ReturnType<typeof mutators>;
constructor(options: TenantContextOptions) {
super({ ...options, lang: options.tenant.locale });
this.tenant = options.tenant;
this.tenantCache = options.tenantCache;
this.user = options.user;
this.mongo = options.mongo;
this.redis = options.redis;
this.scraperQueue = options.scraperQueue;
this.mailerQueue = options.mailerQueue;
this.signingConfig = options.signingConfig;
this.clientID = options.clientID;
this.publisher = createPublisher(
this.pubsub,
this.tenant.id,
this.clientID
);
this.loaders = loaders(this);
this.mutators = mutators(this);
}
+8 -2
View File
@@ -8,7 +8,13 @@ export default (ctx: TenantContext) => ({
discoverOIDCConfiguration: new DataLoader<
string,
GQLDiscoveredOIDCConfiguration | null
>(issuers =>
Promise.all(issuers.map(issuer => discoverOIDCConfiguration(issuer)))
>(
issuers =>
Promise.all(issuers.map(issuer => discoverOIDCConfiguration(issuer))),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
});
@@ -50,10 +50,12 @@ const tagFilter = (tag?: GQLTAG): CommentConnectionInput["filter"] => {
const primeCommentsFromConnection = (ctx: Context) => (
connection: Readonly<Connection<Readonly<Comment>>>
) => {
// For each of the nodes, prime the comment loader.
connection.nodes.forEach(comment => {
ctx.loaders.Comments.comment.prime(comment.id, comment);
});
if (!ctx.disableCaching) {
// For each of the nodes, prime the comment loader.
connection.nodes.forEach(comment => {
ctx.loaders.Comments.comment.prime(comment.id, comment);
});
}
return connection;
};
@@ -96,10 +98,16 @@ const mapVisibleComments = (user?: Pick<User, "role">) => (
): Array<Readonly<Comment> | null> => comments.map(mapVisibleComment(user));
export default (ctx: Context) => ({
comment: new DataLoader((ids: string[]) =>
retrieveManyComments(ctx.mongo, ctx.tenant.id, ids).then(
mapVisibleComments(ctx.user)
)
comment: new DataLoader(
(ids: string[]) =>
retrieveManyComments(ctx.mongo, ctx.tenant.id, ids).then(
mapVisibleComments(ctx.user)
),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
forFilter: ({
first = 10,
@@ -217,13 +225,19 @@ export default (ctx: Context) => ({
// The cursor passed here is always going to be a number.
before: before as number,
}).then(primeCommentsFromConnection(ctx)),
sharedModerationQueueQueuesCounts: new SingletonResolver(() =>
retrieveSharedModerationQueueQueuesCounts(
ctx.mongo,
ctx.redis,
ctx.tenant.id,
ctx.now
)
sharedModerationQueueQueuesCounts: new SingletonResolver(
() =>
retrieveSharedModerationQueueQueuesCounts(
ctx.mongo,
ctx.redis,
ctx.tenant.id,
ctx.now
),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cacheable: !ctx.disableCaching,
}
),
tagCounts: new DataLoader((storyIDs: string[]) =>
retrieveStoryCommentTagCounts(ctx.mongo, ctx.tenant.id, storyIDs)
@@ -56,10 +56,12 @@ const queryFilter = (query?: string): StoryConnectionInput["filter"] => {
const primeStoriesFromConnection = (ctx: TenantContext) => (
connection: Readonly<Connection<Readonly<Story>>>
) => {
// For each of these nodes, prime the story loader.
connection.nodes.forEach(story => {
ctx.loaders.Stories.story.prime(story.id, story);
});
if (!ctx.disableCaching) {
// For each of these nodes, prime the story loader.
connection.nodes.forEach(story => {
ctx.loaders.Stories.story.prime(story.id, story);
});
}
return connection;
};
@@ -72,6 +74,9 @@ export default (ctx: TenantContext) => ({
{
// TODO: (wyattjoh) see if there's something we can do to improve the cache key
cacheKeyFn: (input: FindOrCreateStory) => `${input.id}:${input.url}`,
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
find: new DataLoader(
@@ -81,10 +86,18 @@ export default (ctx: TenantContext) => ({
{
// TODO: (wyattjoh) see if there's something we can do to improve the cache key
cacheKeyFn: (input: FindStory) => `${input.id}:${input.url}`,
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
story: new DataLoader<string, Story | null>(ids =>
retrieveManyStories(ctx.mongo, ctx.tenant.id, ids)
story: new DataLoader<string, Story | null>(
ids => retrieveManyStories(ctx.mongo, ctx.tenant.id, ids),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
connection: ({ first = 10, after, status, query }: QueryToStoriesArgs) =>
retrieveStoryConnection(ctx.mongo, ctx.tenant.id, {
@@ -99,6 +112,11 @@ export default (ctx: TenantContext) => ({
},
}).then(primeStoriesFromConnection(ctx)),
debugScrapeMetadata: new DataLoader(
createManyBatchLoadFn((url: string) => scraper.scrape(url))
createManyBatchLoadFn((url: string) => scraper.scrape(url)),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
),
});
+14 -7
View File
@@ -82,20 +82,27 @@ const statusFilter = (
const primeUsersFromConnection = (ctx: Context) => (
connection: Readonly<Connection<Readonly<User>>>
) => {
// For each of the nodes, prime the user loader.
connection.nodes.forEach(user => {
ctx.loaders.Users.user.prime(user.id, user);
});
if (!ctx.disableCaching) {
// For each of the nodes, prime the user loader.
connection.nodes.forEach(user => {
ctx.loaders.Users.user.prime(user.id, user);
});
}
return connection;
};
export default (ctx: Context) => {
const user = new DataLoader<string, User | null>(ids =>
retrieveManyUsers(ctx.mongo, ctx.tenant.id, ids)
const user = new DataLoader<string, User | null>(
ids => retrieveManyUsers(ctx.mongo, ctx.tenant.id, ids),
{
// Disable caching for the DataLoader if the Context is designed to be
// long lived.
cache: !ctx.disableCaching,
}
);
if (ctx.user) {
if (ctx.user && !ctx.disableCaching) {
// Prime the current logged in user in the dataloader cache.
user.prime(ctx.user.id, ctx.user);
}
+14 -1
View File
@@ -1,15 +1,28 @@
export interface SingletonResolverOptions {
cacheable?: boolean;
}
/**
* SingletonResolver is a cached loader for a single result.
*/
export class SingletonResolver<T> {
private cache: Promise<T> | null = null;
private resolver: () => Promise<T>;
private cacheable: boolean;
constructor(resolver: () => Promise<T>) {
constructor(
resolver: () => Promise<T>,
{ cacheable = true }: SingletonResolverOptions = {}
) {
this.resolver = resolver;
this.cacheable = cacheable;
}
public load() {
if (!this.cacheable) {
return this.resolver();
}
if (this.cache) {
return this.cache;
}
@@ -7,13 +7,13 @@ import {
export const Actions = (ctx: TenantContext) => ({
approveComment: (input: GQLApproveCommentInput) =>
approve(ctx.mongo, ctx.redis, ctx.tenant, {
approve(ctx.mongo, ctx.redis, ctx.publisher, ctx.tenant, {
commentID: input.commentID,
commentRevisionID: input.commentRevisionID,
moderatorID: ctx.user!.id,
}),
rejectComment: (input: GQLRejectCommentInput) =>
reject(ctx.mongo, ctx.redis, ctx.tenant, {
reject(ctx.mongo, ctx.redis, ctx.publisher, ctx.tenant, {
commentID: input.commentID,
commentRevisionID: input.commentRevisionID,
moderatorID: ctx.user!.id,
@@ -43,6 +43,7 @@ export const Comments = (ctx: TenantContext) => ({
create(
ctx.mongo,
ctx.redis,
ctx.publisher,
ctx.tenant,
ctx.user!,
{ authorID: ctx.user!.id, ...comment },
@@ -64,6 +65,7 @@ export const Comments = (ctx: TenantContext) => ({
edit(
ctx.mongo,
ctx.redis,
ctx.publisher,
ctx.tenant,
ctx.user!,
{
@@ -87,6 +89,7 @@ export const Comments = (ctx: TenantContext) => ({
createReaction(
ctx.mongo,
ctx.redis,
ctx.publisher,
ctx.tenant,
ctx.user!,
{
@@ -107,6 +110,7 @@ export const Comments = (ctx: TenantContext) => ({
createDontAgree(
ctx.mongo,
ctx.redis,
ctx.publisher,
ctx.tenant,
ctx.user!,
{
@@ -133,6 +137,7 @@ export const Comments = (ctx: TenantContext) => ({
createFlag(
ctx.mongo,
ctx.redis,
ctx.publisher,
ctx.tenant,
ctx.user!,
{
@@ -161,7 +166,7 @@ export const Comments = (ctx: TenantContext) => ({
ctx.now
).then(comment =>
comment.status !== GQLCOMMENT_STATUS.APPROVED
? approve(ctx.mongo, ctx.redis, ctx.tenant, {
? approve(ctx.mongo, ctx.redis, ctx.publisher, ctx.tenant, {
commentID,
commentRevisionID,
moderatorID: ctx.user!.id,
@@ -0,0 +1,38 @@
import {
GQLMODERATION_QUEUE,
SubscriptionToCommentEnteredModerationQueueResolver,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { createIterator } from "./helpers";
import { SUBSCRIPTION_CHANNELS, SubscriptionPayload } from "./types";
export interface CommentEnteredModerationQueueInput
extends SubscriptionPayload {
queue: GQLMODERATION_QUEUE;
commentID: string;
storyID: string;
}
export const commentEnteredModerationQueue: SubscriptionToCommentEnteredModerationQueueResolver<
CommentEnteredModerationQueueInput
> = createIterator(SUBSCRIPTION_CHANNELS.COMMENT_ENTERED_MODERATION_QUEUE, {
filter: (source, { storyID, queue }) => {
// If we're filtering by storyID, then only send back comments with the
// specific storyID.
if (storyID && source.storyID !== storyID) {
return false;
}
// If we're filtering by queue, then only send back comments from the
// specific queue.
if (queue && source.queue !== queue) {
return false;
}
return true;
},
resolve: ({ queue, commentID }, args, ctx) => ({
queue: () => queue,
comment: () => ctx.loaders.Comments.comment.load(commentID),
}),
});
@@ -0,0 +1,37 @@
import {
GQLMODERATION_QUEUE,
SubscriptionToCommentLeftModerationQueueResolver,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { createIterator } from "./helpers";
import { SUBSCRIPTION_CHANNELS, SubscriptionPayload } from "./types";
export interface CommentLeftModerationQueueInput extends SubscriptionPayload {
queue: GQLMODERATION_QUEUE;
commentID: string;
storyID: string;
}
export const commentLeftModerationQueue: SubscriptionToCommentLeftModerationQueueResolver<
CommentLeftModerationQueueInput
> = createIterator(SUBSCRIPTION_CHANNELS.COMMENT_LEFT_MODERATION_QUEUE, {
filter: (source, { storyID, queue }) => {
// If we're filtering by storyID, then only send back comments with the
// specific storyID.
if (storyID && source.storyID !== storyID) {
return false;
}
// If we're filtering by queue, then only send back comments from the
// specific queue.
if (queue && source.queue !== queue) {
return false;
}
return true;
},
resolve: ({ queue, commentID }, args, ctx) => ({
queue: () => queue,
comment: () => ctx.loaders.Comments.comment.load(commentID),
}),
});
@@ -0,0 +1,40 @@
import {
GQLCOMMENT_STATUS,
SubscriptionToCommentStatusUpdatedResolver,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { createIterator } from "./helpers";
import { SUBSCRIPTION_CHANNELS, SubscriptionPayload } from "./types";
export interface CommentStatusUpdatedInput extends SubscriptionPayload {
newStatus: GQLCOMMENT_STATUS;
oldStatus: GQLCOMMENT_STATUS;
moderatorID: string | null;
commentID: string;
}
export const commentStatusUpdated: SubscriptionToCommentStatusUpdatedResolver<
CommentStatusUpdatedInput
> = createIterator(SUBSCRIPTION_CHANNELS.COMMENT_STATUS_UPDATED, {
filter: (source, { id }) => {
// If we're filtering by id, then only send back updates for the specified
// comment.
if (id && source.commentID !== id) {
return false;
}
return true;
},
resolve: ({ newStatus, oldStatus, moderatorID, commentID }, args, ctx) => ({
newStatus: () => newStatus,
oldStatus: () => oldStatus,
moderator: () => {
if (moderatorID) {
return ctx.loaders.Users.user.load(moderatorID);
}
return null;
},
comment: () => ctx.loaders.Comments.comment.load(commentID),
}),
});
@@ -0,0 +1,108 @@
import { GraphQLResolveInfo } from "graphql";
import { withFilter } from "graphql-subscriptions";
import TenantContext from "../../context";
import { SUBSCRIPTION_CHANNELS, SubscriptionPayload } from "./types";
type FilterFn<TParent, TArgs, TContext> = (
parent: TParent,
args: TArgs,
context: TContext,
info: GraphQLResolveInfo
) => boolean | Promise<boolean>;
type Resolver<TParent, TArgs, TResult> = (
source: TParent,
args: TArgs,
ctx: TenantContext,
info: GraphQLResolveInfo
) => TResult;
interface SubscriptionResolver<TParent, TArgs, TResult> {
subscribe: Resolver<TParent, TArgs, AsyncIterator<TResult>>;
resolve?: Resolver<TParent, TArgs, TResult>;
}
export function createTenantAsyncIterator<TParent, TArgs, TResult>(
channel: SUBSCRIPTION_CHANNELS
): Resolver<TParent, TArgs, AsyncIterator<TResult>> {
return (source, args, ctx) =>
ctx.pubsub.asyncIterator<TResult>(
createSubscriptionChannelName(ctx.tenant.id, channel)
);
}
export function createSubscriptionChannelName(
tenantID: string,
channel: SUBSCRIPTION_CHANNELS
): string {
return `TENANT[${tenantID}][${channel}]`;
}
/**
* defaultFilterFn will perform filtering operations on the subscription
* responses to ensure that mutations issued by one user is not sent back as a
* subscription to the same requesting User, as they already implement the
* update via the mutation response.
*
* @param source the source for the document passed down, we don't actually need
* it here.
* @param args the arguments for the specific subscription operation, we don't
* actually need it here.
* @param ctx the context for the request, this contains the references we'll
* need to determine eligibility to send the subscription back or
* not.
*/
export function defaultFilterFn<TParent extends SubscriptionPayload, TArgs>(
source: TParent,
args: TArgs,
ctx: TenantContext
): boolean {
if (source.clientID && ctx.clientID && source.clientID === ctx.clientID) {
return false;
}
return true;
}
/**
* Ensure that even when we're provided with a domain specific filtering
* function we respect the subscription id that is sent back with the request to
* prevent double responses.
*/
export function createFilterFn<TParent, TArgs>(
filter?: FilterFn<TParent, TArgs, TenantContext>
): FilterFn<TParent, TArgs, TenantContext> {
return filter
? // Combine the filters, preferring the defaultFilterFn first.
(source, args, ctx, info) => {
if (!defaultFilterFn(source, args, ctx)) {
return false;
}
return filter(source, args, ctx, info);
}
: defaultFilterFn;
}
export interface CreateIteratorInput<TParent, TArgs, TResult> {
filter?: FilterFn<TParent, TArgs, TenantContext>;
resolve?: Resolver<TParent, TArgs, TResult>;
}
export function createIterator<
TParent extends SubscriptionPayload,
TArgs,
TResult
>(
channel: SUBSCRIPTION_CHANNELS,
{ filter, resolve }: CreateIteratorInput<TParent, TArgs, TResult> = {}
): SubscriptionResolver<TParent, TArgs, TResult> {
return {
subscribe: withFilter(
createTenantAsyncIterator(channel),
createFilterFn(filter)
),
resolve,
};
}
@@ -0,0 +1,11 @@
import { GQLSubscriptionTypeResolver } from "coral-server/graph/tenant/schema/__generated__/types";
import { commentEnteredModerationQueue } from "./commentEnteredModerationQueue";
import { commentLeftModerationQueue } from "./commentLeftModerationQueue";
import { commentStatusUpdated } from "./commentStatusUpdated";
export const Subscription: GQLSubscriptionTypeResolver = {
commentEnteredModerationQueue,
commentLeftModerationQueue,
commentStatusUpdated,
};
@@ -0,0 +1,35 @@
import { CommentEnteredModerationQueueInput } from "./commentEnteredModerationQueue";
import { CommentLeftModerationQueueInput } from "./commentLeftModerationQueue";
import { CommentStatusUpdatedInput } from "./commentStatusUpdated";
export enum SUBSCRIPTION_CHANNELS {
COMMENT_ENTERED_MODERATION_QUEUE = "COMMENT_ENTERED_MODERATION_QUEUE",
COMMENT_LEFT_MODERATION_QUEUE = "COMMENT_LEFT_MODERATION_QUEUE",
COMMENT_STATUS_UPDATED = "COMMENT_STATUS_UPDATED",
}
export interface SubscriptionPayload {
clientID?: string;
}
export interface SubscriptionType<
TChannel extends SUBSCRIPTION_CHANNELS,
TPayload extends SubscriptionPayload
> {
channel: TChannel;
payload: TPayload;
}
export type SUBSCRIPTION_INPUT =
| SubscriptionType<
SUBSCRIPTION_CHANNELS.COMMENT_ENTERED_MODERATION_QUEUE,
CommentEnteredModerationQueueInput
>
| SubscriptionType<
SUBSCRIPTION_CHANNELS.COMMENT_LEFT_MODERATION_QUEUE,
CommentLeftModerationQueueInput
>
| SubscriptionType<
SUBSCRIPTION_CHANNELS.COMMENT_STATUS_UPDATED,
CommentStatusUpdatedInput
>;
@@ -25,6 +25,7 @@ import { Query } from "./Query";
import { RejectCommentPayload } from "./RejectCommentPayload";
import { Story } from "./Story";
import { StorySettings } from "./StorySettings";
import { Subscription } from "./Subscription";
import { SuspensionStatus } from "./SuspensionStatus";
import { SuspensionStatusHistory } from "./SuspensionStatusHistory";
import { Tag } from "./Tag";
@@ -56,6 +57,7 @@ const Resolvers: GQLResolver = {
RejectCommentPayload,
Story,
StorySettings,
Subscription,
SuspensionStatus,
SuspensionStatusHistory,
Tag,
@@ -4459,3 +4459,121 @@ type Mutation {
removeUserIgnore(input: RemoveUserIgnoreInput!): RemoveUserIgnorePayload!
@auth(permit: [SUSPENDED, BANNED])
}
##################
## Subscriptions
##################
"""
CommentStatusUpdatedPayload is returned when a Comment has it's status updated
after it was created.
"""
type CommentStatusUpdatedPayload {
"""
newStatus is the new status assigned to the Comment. This status may not
match the status provided by `comment.status` due to race conditions in the
data loaders.
"""
newStatus: COMMENT_STATUS!
"""
oldStatus is the old status that was previously assigned to the Comment.
"""
oldStatus: COMMENT_STATUS!
"""
moderator is the User that updated the Comment's status. If null, then the
system assigned the new Comment status (for example, when a comment is edited
by the author, and now contains a banned word).
"""
moderator: User
"""
comment is the updated Comment after the status has been updated.
"""
comment: Comment!
}
"""
MODERATION_QUEUE references the specific ModerationQueue that a given Comment
can be associated with.
"""
enum MODERATION_QUEUE {
"""
UNMODERATED refers to the ModerationQueue for all Comments that have not been
moderated yet.
"""
UNMODERATED
"""
REPORTED refers to the ModerationQueue for all Comments that have been
published, have not been moderated by a human yet, and have been reported by
a User via a flag.
"""
REPORTED
"""
PENDING refers to the ModerationQueue for all Comments that were held back by
the system and require moderation in order to be published.
"""
PENDING
}
"""
CommentEnteredModerationQueuePayload is returned when a Comment enters a
specific ModerationQueue.
"""
type CommentEnteredModerationQueuePayload {
"""
queue refers to the specific ModerationQueue that a given Comment entered.
"""
queue: MODERATION_QUEUE!
"""
comment is the Comment that entered the ModerationQueue.
"""
comment: Comment!
}
"""
CommentLeftModerationQueuePayload is returned when a Comment leaves a specific
ModerationQueue.
"""
type CommentLeftModerationQueuePayload {
"""
queue refers to the specific ModerationQueue that a given Comment left.
"""
queue: MODERATION_QUEUE!
"""
comment is the Comment that left the ModerationQueue.
"""
comment: Comment!
}
type Subscription {
"""
commentEnteredModerationQueue returns when a Comment enters a ModerationQueue.
Note that a Comment may enter multiple moderation queues.
"""
commentEnteredModerationQueue(
storyID: ID
queue: MODERATION_QUEUE
): CommentEnteredModerationQueuePayload! @auth(roles: [MODERATOR, ADMIN])
"""
commentLeftModerationQueue returns when a Comment leaves a ModerationQueue.
Note that a Comment may leave multiple moderation queues.
"""
commentLeftModerationQueue(
storyID: ID
queue: MODERATION_QUEUE
): CommentLeftModerationQueuePayload! @auth(roles: [MODERATOR, ADMIN])
"""
commentStatusUpdated returns when a Comment has it's status changed after
being created.
"""
commentStatusUpdated(id: ID): CommentStatusUpdatedPayload!
@auth(roles: [MODERATOR, ADMIN])
}
@@ -0,0 +1,28 @@
import { RedisPubSub } from "graphql-redis-subscriptions";
import { createSubscriptionChannelName } from "coral-server/graph/tenant/resolvers/Subscription/helpers";
import { SUBSCRIPTION_INPUT } from "coral-server/graph/tenant/resolvers/Subscription/types";
import logger from "coral-server/logger";
export type Publisher = (input: SUBSCRIPTION_INPUT) => Promise<void>;
/**
* createPublisher will create a new Publisher that can be used to send events
* over the pubsub broker to facilitate live updates.
*
* @param pubsub the pubsub broker to be used to facilitate the publish action
* @param tenantID the ID of the Tenant where the event will be published with
* @param clientID the ID of the client to de-duplicate mutation responses
*/
export const createPublisher = (
pubsub: RedisPubSub,
tenantID: string,
clientID?: string
): Publisher => async ({ channel, payload }) => {
logger.trace({ channel, tenantID, clientID }, "publishing event");
return pubsub.publish(createSubscriptionChannelName(tenantID, channel), {
...payload,
clientID,
});
};
@@ -0,0 +1,222 @@
import {
execute,
ExecutionResult,
GraphQLSchema,
parse,
subscribe,
} from "graphql";
import http, { IncomingMessage } from "http";
import {
ConnectionContext,
ExecutionParams,
OperationMessagePayload,
SubscriptionServer,
} from "subscriptions-transport-ws";
import { ACCESS_TOKEN_PARAM, CLIENT_ID_PARAM } from "coral-common/constants";
import { Omit } from "coral-common/types";
import { AppOptions } from "coral-server/app";
import { getHostname } from "coral-server/app/helpers/hostname";
import {
createVerifiers,
verifyAndRetrieveUser,
} from "coral-server/app/middleware/passport/strategies/jwt";
import {
CoralError,
InternalError,
TenantNotFoundError,
} from "coral-server/errors";
import {
enrichError,
logError,
logQuery,
} from "coral-server/graph/common/extensions";
import { getOperationMetadata } from "coral-server/graph/common/extensions/helpers";
import logger from "coral-server/logger";
import { extractTokenFromRequest } from "coral-server/services/jwt";
import TenantContext, { TenantContextOptions } from "../context";
type OnConnectFn = (
params: OperationMessagePayload,
socket: any,
context: ConnectionContext
) => Promise<TenantContext>;
export function extractTokenFromWSRequest(
connectionParams: OperationMessagePayload,
req: IncomingMessage
): string | null {
// Try to grab the token from the connection params if available.
if (
typeof connectionParams[ACCESS_TOKEN_PARAM] === "string" &&
connectionParams[ACCESS_TOKEN_PARAM].length > 0
) {
return connectionParams[ACCESS_TOKEN_PARAM];
}
// Try to get the access token from the request.
return extractTokenFromRequest(req);
}
export function extractClientID(connectionParams: OperationMessagePayload) {
if (
typeof connectionParams[CLIENT_ID_PARAM] === "string" &&
connectionParams[CLIENT_ID_PARAM].length > 0
) {
return connectionParams[CLIENT_ID_PARAM];
}
return null;
}
export type OnConnectOptions = Omit<
TenantContextOptions,
"tenant" | "signingConfig" | "disableCaching"
> &
Required<Pick<TenantContextOptions, "signingConfig">>;
export function onConnect(options: OnConnectOptions): OnConnectFn {
// Create the JWT verifiers that will be used to verify all the requests
// coming in.
const verifiers = createVerifiers(options);
// Return the per-connection operation.
return async (connectionParams, socket) => {
try {
// Pull the upgrade request off of the connection.
const req: IncomingMessage = socket.upgradeReq;
// Get the hostname of the request.
const hostname = getHostname(req);
if (!hostname) {
throw new Error("could not detect hostname");
}
// Get the Tenant for this hostname.
const tenant = await options.tenantCache.retrieveByDomain(hostname);
if (!tenant) {
throw new TenantNotFoundError(hostname);
}
// Create some new options to store the tenant context details inside.
const opts: TenantContextOptions = {
...options,
// Disable caching with this Context to ensure that every call (besides)
// to the tenant, is not cached, and is instead fresh.
disableCaching: true,
tenant,
};
// If the token is available, try to get the user.
const tokenString = extractTokenFromWSRequest(connectionParams, req);
if (tokenString) {
const user = await verifyAndRetrieveUser(
verifiers,
tenant,
tokenString
);
if (user) {
opts.user = user;
}
}
// Extract the users clientID from the request.
const clientID = extractClientID(connectionParams);
if (clientID) {
opts.clientID = clientID;
}
return new TenantContext(opts);
} catch (err) {
logger.error({ err }, "could not setup websocket connection");
if (!(err instanceof CoralError)) {
err = new InternalError(err, "could not setup websocket connection");
}
const { message } = err.serializeExtensions(
options.i18n.getDefaultBundle()
);
throw { message };
}
};
}
export type FormatResponseOptions = Pick<AppOptions, "metrics">;
export function formatResponse({ metrics }: FormatResponseOptions) {
return (
value: ExecutionResult,
{ context, query }: ExecutionParams<TenantContext>
) => {
// Parse the query in order to extract operation metadata.
if (typeof query === "string") {
query = parse(query);
}
// Log out the query.
logQuery(context, query);
// Increment the metrics if enabled.
if (metrics) {
// Get the request metadata.
const { operation, operationName } = getOperationMetadata(query);
if (operation && operationName) {
// Increment the graph query value, tagging with the name of the query.
metrics.executedGraphQueriesTotalCounter
.labels(operation, operationName)
.inc();
}
}
if (value.errors && value.errors.length > 0) {
return {
...value,
errors: value.errors.map(err => {
const enriched = enrichError(context, err);
// Log the error out.
logError(context, enriched);
return enriched;
}),
};
}
return value;
};
}
export type OnOperationOptions = FormatResponseOptions;
export function onOperation(options: OnOperationOptions) {
return (message: any, params: ExecutionParams<TenantContext>) => {
// Attach the response formatter.
params.formatResponse = formatResponse(options);
return params;
};
}
export type Options = OnConnectOptions & OnOperationOptions;
export function createSubscriptionServer(
server: http.Server,
schema: GraphQLSchema,
options: Options
) {
return SubscriptionServer.create(
{
schema,
execute,
subscribe,
onConnect: onConnect(options),
onOperation: onOperation(options),
},
{
server,
path: "/api/graphql/live",
}
);
}
+42 -15
View File
@@ -1,18 +1,28 @@
import cluster from "cluster";
import express, { Express } from "express";
import { GraphQLSchema } from "graphql";
import { RedisPubSub } from "graphql-redis-subscriptions";
import http from "http";
import { Db } from "mongodb";
import { AggregatorRegistry, collectDefaultMetrics } from "prom-client";
import { SubscriptionServer } from "subscriptions-transport-ws";
import { LanguageCode } from "coral-common/helpers/i18n/locales";
import { createApp, listenAndServe } from "coral-server/app";
import { AppOptions, createApp, listenAndServe } from "coral-server/app";
import { basicAuth } from "coral-server/app/middleware/basicAuth";
import { noCacheMiddleware } from "coral-server/app/middleware/cacheHeaders";
import { JSONErrorHandler } from "coral-server/app/middleware/error";
import { accessLogger, errorLogger } from "coral-server/app/middleware/logging";
import { notFoundMiddleware } from "coral-server/app/middleware/notFound";
import config, { Config } from "coral-server/config";
import { createPubSubClient } from "coral-server/graph/common/subscriptions/pubsub";
import getTenantSchema from "coral-server/graph/tenant/schema";
import { createSubscriptionServer } from "coral-server/graph/tenant/subscriptions/server";
import logger from "coral-server/logger";
import { createQueue, TaskQueue } from "coral-server/queue";
import { I18n } from "coral-server/services/i18n";
import { createJWTSigningConfig } from "coral-server/services/jwt";
import { createMetrics } from "coral-server/services/metrics";
import { createMongoDB } from "coral-server/services/mongodb";
import { ensureIndexes } from "coral-server/services/mongodb/indexes";
import {
@@ -21,11 +31,6 @@ import {
createRedisClient,
} from "coral-server/services/redis";
import TenantCache from "coral-server/services/tenant/cache";
import { basicAuth } from "./app/middleware/basicAuth";
import { noCacheMiddleware } from "./app/middleware/cacheHeaders";
import { JSONErrorHandler } from "./app/middleware/error";
import { accessLogger, errorLogger } from "./app/middleware/logging";
import { notFoundMiddleware } from "./app/middleware/notFound";
export interface ServerOptions {
/**
@@ -55,12 +60,19 @@ class Server {
// the requested port.
public httpServer: http.Server;
// subscriptionServer is the running instance of the HTTP server that will
// bind to the requested port to serve websocket traffic.
public subscriptionServer: SubscriptionServer;
// tasks stores a reference to the queues that can process operations.
private tasks: TaskQueue;
// redis stores the redis connection used by the application.
private redis: AugmentedRedis;
// pubsub stores the pubsub engine used by the application.
private pubsub: RedisPubSub;
// mongo stores the mongo connection used by the application.
private mongo: Db;
@@ -134,6 +146,12 @@ class Server {
i18n: this.i18n,
});
// Create the pubsub client.
this.pubsub = createPubSubClient(
createRedisClient(this.config),
createRedisClient(this.config)
);
// Setup the metrics collectors.
collectDefaultMetrics({ timeout: 5000 });
}
@@ -238,16 +256,13 @@ class Server {
// Create the signing config.
const signingConfig = createJWTSigningConfig(this.config);
// Only enable the metrics server if concurrency is set to 1.
const metrics = this.config.get("concurrency") === 1;
// Disables the client routes to serve bundles etc. Useful for devleoping with
// Disables the client routes to serve bundles etc. Useful for developing with
// Webpack Dev Server.
const disableClientRoutes = this.config.get("disable_client_routes");
// Create the Coral App, branching off from the parent app.
const app: Express = await createApp({
const options: AppOptions = {
parent,
pubsub: this.pubsub,
mongo: this.mongo,
redis: this.redis,
signingConfig,
@@ -257,16 +272,28 @@ class Server {
i18n: this.i18n,
mailerQueue: this.tasks.mailer,
scraperQueue: this.tasks.scraper,
metrics,
disableClientRoutes,
});
};
// Only enable the metrics server if concurrency is set to 1.
if (this.config.get("concurrency") === 1) {
options.metrics = createMetrics();
}
// Create the Coral App, branching off from the parent app.
const app: Express = await createApp(options);
// Start the application and store the resulting http.Server. The server
// will return when the server starts listening. The NodeJS application will
// not exit until all tasks are handled, which for an open socket, is never.
this.httpServer = await listenAndServe(app, port);
// TODO: (wyattjoh) add the subscription handler here
// Setup subscriptions and attach it to the httpServer.
this.subscriptionServer = createSubscriptionServer(
this.httpServer,
this.schema,
options
);
logger.info({ port }, "now listening");
}
+1 -1
View File
@@ -11,6 +11,6 @@ export function roleIsStaff(role: GQLUSER_ROLE) {
return false;
}
export function userIsStaff(user: User) {
export function userIsStaff(user: Pick<User, "role">) {
return roleIsStaff(user.role);
}
+19 -6
View File
@@ -1,7 +1,9 @@
import { Db } from "mongodb";
import { Omit } from "coral-common/types";
import { CommentNotFoundError } from "coral-server/errors";
import { GQLCOMMENT_FLAG_REPORTED_REASON } from "coral-server/graph/tenant/schema/__generated__/types";
import { Publisher } from "coral-server/graph/tenant/subscriptions/publisher";
import {
ACTION_TYPE,
CommentAction,
@@ -25,8 +27,9 @@ import {
} from "coral-server/models/story";
import { Tenant } from "coral-server/models/tenant";
import { User } from "coral-server/models/user";
import { publishModerationQueueChanges } from "coral-server/services/events";
import { AugmentedRedis } from "coral-server/services/redis";
import { AugmentedRedis } from "../redis";
import { calculateCountsDiff } from "./moderation/counts";
export type CreateAction = CreateActionInput;
@@ -78,14 +81,14 @@ export async function addCommentActionCounts(
async function addCommentAction(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
input: Omit<CreateActionInput, "storyID">,
now = new Date()
): Promise<Readonly<Comment>> {
const oldComment = await retrieveComment(mongo, tenant.id, input.commentID);
if (!oldComment) {
// TODO: replace to match error returned by the models/comments.ts
throw new Error("comment not found");
throw new CommentNotFoundError(input.commentID);
}
// Create the action creator input.
@@ -105,12 +108,17 @@ async function addCommentAction(
...commentActions
);
const moderationQueue = calculateCountsDiff(oldComment, updatedComment);
// Calculate the new story counts.
await updateStoryCounts(mongo, redis, tenant.id, updatedComment.storyID, {
action: encodeActionCounts(...commentActions),
moderationQueue: calculateCountsDiff(oldComment, updatedComment),
moderationQueue,
});
// Publish changes to the queue.
publishModerationQueueChanges(publish, moderationQueue, updatedComment);
return updatedComment;
}
@@ -126,8 +134,7 @@ export async function removeCommentAction(
// Get the Comment that we are leaving the Action on.
const comment = await retrieveComment(mongo, tenant.id, input.commentID);
if (!comment) {
// TODO: replace to match error returned by the models/comments.ts
throw new Error("comment not found");
throw new CommentNotFoundError(input.commentID);
}
// Get the revision for the specific action being removed.
@@ -198,6 +205,7 @@ export type CreateCommentReaction = Pick<
export async function createReaction(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
author: User,
input: CreateCommentReaction,
@@ -206,6 +214,7 @@ export async function createReaction(
return addCommentAction(
mongo,
redis,
publish,
tenant,
{
actionType: ACTION_TYPE.REACTION,
@@ -241,6 +250,7 @@ export type CreateCommentDontAgree = Pick<
export async function createDontAgree(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
author: User,
input: CreateCommentDontAgree,
@@ -249,6 +259,7 @@ export async function createDontAgree(
return addCommentAction(
mongo,
redis,
publish,
tenant,
{
actionType: ACTION_TYPE.DONT_AGREE,
@@ -287,6 +298,7 @@ export type CreateCommentFlag = Pick<
export async function createFlag(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
author: User,
input: CreateCommentFlag,
@@ -295,6 +307,7 @@ export async function createFlag(
return addCommentAction(
mongo,
redis,
publish,
tenant,
{
actionType: ACTION_TYPE.FLAG,
+56 -22
View File
@@ -1,12 +1,21 @@
import { DateTime } from "luxon";
import { Db } from "mongodb";
import { ERROR_TYPES } from "coral-common/errors";
import { Omit } from "coral-common/types";
import {
CommentNotFoundError,
CoralError,
StoryNotFoundError,
} from "coral-server/errors";
import { GQLTAG } from "coral-server/graph/tenant/schema/__generated__/types";
import { Publisher } from "coral-server/graph/tenant/subscriptions/publisher";
import logger from "coral-server/logger";
import {
encodeActionCounts,
filterDuplicateActions,
} from "coral-server/models/action/comment";
import { createCommentModerationAction } from "coral-server/models/action/moderation/comment";
import {
addCommentTag,
createComment,
@@ -19,6 +28,10 @@ import {
retrieveComment,
validateEditable,
} from "coral-server/models/comment";
import {
hasAncestors,
hasVisibleStatus,
} from "coral-server/models/comment/helpers";
import {
retrieveStory,
StoryCounts,
@@ -26,20 +39,13 @@ import {
} from "coral-server/models/story";
import { Tenant } from "coral-server/models/tenant";
import { User } from "coral-server/models/user";
import {
publishCommentStatusChanges,
publishModerationQueueChanges,
} from "coral-server/services/events";
import { AugmentedRedis } from "coral-server/services/redis";
import { Request } from "coral-server/types/express";
import { ERROR_TYPES } from "coral-common/errors";
import {
CommentNotFoundError,
CoralError,
StoryNotFoundError,
} from "coral-server/errors";
import { GQLTAG } from "coral-server/graph/tenant/schema/__generated__/types";
import {
hasAncestors,
hasVisibleStatus,
} from "coral-server/models/comment/helpers";
import { AugmentedRedis } from "../redis";
import { addCommentActions, CreateAction } from "./actions";
import { calculateCounts, calculateCountsDiff } from "./moderation/counts";
import { PhaseResult, processForModeration } from "./pipeline";
@@ -52,6 +58,7 @@ export type CreateComment = Omit<
export async function create(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
author: User,
input: CreateComment,
@@ -205,6 +212,10 @@ export async function create(
log.trace({ actions: upsertedActions.length }, "added actions to comment");
}
const moderationQueue = calculateCounts(comment);
// Publish changes to the queue.
publishModerationQueueChanges(publish, moderationQueue, comment);
// Compile the changes we want to apply to the story counts.
const storyCounts: Required<Omit<StoryCounts, "action">> = {
@@ -212,7 +223,7 @@ export async function create(
status: { [status]: 1 },
// This comment is being created, so we can compute it raw from the comment
// that we created.
moderationQueue: calculateCounts(comment),
moderationQueue,
};
log.trace({ storyCounts }, "updating story status counts");
@@ -231,6 +242,7 @@ export type EditComment = Omit<
export async function edit(
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
author: User,
input: EditComment,
@@ -346,19 +358,21 @@ export async function edit(
);
}
// Compute the changes in queue counts. This looks at the action counts that
// are encoded, as well as the comment status's. We however may have had the
// comment status when we grabbed the updated comment after changing the
// action counts, so we extract the action counts out of the edited comment
// and use the status from the moderation decision.
const moderationQueue = calculateCountsDiff(oldComment, {
status,
actionCounts: editedComment.actionCounts,
});
// Compile the changes we want to apply to the story counts.
const storyCounts: Required<Omit<StoryCounts, "action">> = {
// Status is updated below if it has been changed.
status: {},
// Compute the changes in queue counts. This looks at the action counts that
// are encoded, as well as the comment status's. We however may have had the
// comment status when we grabbed the updated comment after changing the
// action counts, so we extract the action counts out of the edited comment
// and use the status from the moderation decision.
moderationQueue: calculateCountsDiff(oldComment, {
status,
actionCounts: editedComment.actionCounts,
}),
moderationQueue,
};
if (oldComment.status !== editedComment.status) {
@@ -368,6 +382,15 @@ export async function edit(
// on the moderation pipeline.
storyCounts.status[oldComment.status] = -1;
storyCounts.status[status] = 1;
// The comment status changed as a result of a pipeline operation, create a
// moderation action as a result.
await createCommentModerationAction(mongo, tenant.id, {
commentID: editedComment.id,
commentRevisionID: newRevision.id,
status: editedComment.status,
moderatorID: null,
});
}
log.trace({ storyCounts }, "updating story status counts");
@@ -375,6 +398,17 @@ export async function edit(
// Update the story counts as a result.
await updateStoryCounts(mongo, redis, tenant.id, story.id, storyCounts);
// Publish changes.
publishModerationQueueChanges(publish, moderationQueue, editedComment);
publishCommentStatusChanges(
publish,
oldComment.status,
editedComment.status,
editedComment.id,
// This is a comment that was edited, so it should not present a moderator.
null
);
return editedComment;
}
@@ -1,7 +1,9 @@
import { Db } from "mongodb";
import { Omit } from "coral-common/types";
import { CommentNotFoundError, StoryNotFoundError } from "coral-server/errors";
import { GQLCOMMENT_STATUS } from "coral-server/graph/tenant/schema/__generated__/types";
import { Publisher } from "coral-server/graph/tenant/subscriptions/publisher";
import logger from "coral-server/logger";
import {
createCommentModerationAction,
@@ -10,7 +12,12 @@ import {
import { updateCommentStatus } from "coral-server/models/comment";
import { updateStoryCounts } from "coral-server/models/story";
import { Tenant } from "coral-server/models/tenant";
import {
publishCommentStatusChanges,
publishModerationQueueChanges,
} from "coral-server/services/events";
import { AugmentedRedis } from "coral-server/services/redis";
import { calculateCountsDiff } from "./counts";
export type Moderate = Omit<CreateCommentModerationActionInput, "status">;
@@ -20,6 +27,7 @@ const moderate = (
) => async (
mongo: Db,
redis: AugmentedRedis,
publish: Publisher,
tenant: Tenant,
input: Moderate
) => {
@@ -41,8 +49,7 @@ const moderate = (
status
);
if (!result) {
// TODO: wrap in better error?
throw new Error("specified comment not found");
throw new CommentNotFoundError(input.commentID, input.commentRevisionID);
}
log.trace("updated comment status");
@@ -62,6 +69,19 @@ const moderate = (
"created the moderation action"
);
// Compute the queue difference as a result of the old status and the new
// status.
const moderationQueue = calculateCountsDiff(
{
status: result.oldStatus,
actionCounts: result.comment.actionCounts,
},
{
status,
actionCounts: result.comment.actionCounts,
}
);
// Update the story comment counts.
const story = await updateStoryCounts(
mongo,
@@ -74,25 +94,24 @@ const moderate = (
[result.oldStatus]: -1,
[status]: 1,
},
// Compute the queue difference as a result of the old status and the new
// status.
moderationQueue: calculateCountsDiff(
{
status: result.oldStatus,
actionCounts: result.comment.actionCounts,
},
{
status,
actionCounts: result.comment.actionCounts,
}
),
moderationQueue,
}
);
if (!story) {
// TODO: wrap in better error?
throw new Error("specified story not found");
throw new StoryNotFoundError(result.comment.storyID);
}
// Publish changes.
publishModerationQueueChanges(publish, moderationQueue, result.comment);
publishCommentStatusChanges(
publish,
result.oldStatus,
status,
result.comment.id,
input.moderatorID
);
log.trace({ oldStatus: result.oldStatus }, "adjusted story comment counts");
return result.comment;
@@ -0,0 +1,92 @@
import { SUBSCRIPTION_CHANNELS } from "coral-server/graph/tenant/resolvers/Subscription/types";
import {
GQLCOMMENT_STATUS,
GQLMODERATION_QUEUE,
} from "coral-server/graph/tenant/schema/__generated__/types";
import { Publisher } from "coral-server/graph/tenant/subscriptions/publisher";
import { Comment } from "coral-server/models/comment";
import { CommentModerationQueueCounts } from "coral-server/models/story/counts";
export function publishCommentStatusChanges(
publish: Publisher,
oldStatus: GQLCOMMENT_STATUS,
newStatus: GQLCOMMENT_STATUS,
commentID: string,
moderatorID: string | null
) {
if (oldStatus !== newStatus) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_STATUS_UPDATED,
payload: {
newStatus,
oldStatus,
commentID,
moderatorID,
},
});
}
}
export function publishModerationQueueChanges(
publish: Publisher,
moderationQueue: Pick<CommentModerationQueueCounts, "queues">,
comment: Pick<Comment, "id" | "storyID">
) {
if (moderationQueue.queues.pending === 1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_ENTERED_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.PENDING,
commentID: comment.id,
storyID: comment.storyID,
},
});
} else if (moderationQueue.queues.pending === -1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_LEFT_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.PENDING,
commentID: comment.id,
storyID: comment.storyID,
},
});
}
if (moderationQueue.queues.reported === 1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_ENTERED_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.REPORTED,
commentID: comment.id,
storyID: comment.storyID,
},
});
} else if (moderationQueue.queues.reported === -1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_LEFT_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.REPORTED,
commentID: comment.id,
storyID: comment.storyID,
},
});
}
if (moderationQueue.queues.unmoderated === 1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_ENTERED_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.UNMODERATED,
commentID: comment.id,
storyID: comment.storyID,
},
});
} else if (moderationQueue.queues.unmoderated === -1) {
publish({
channel: SUBSCRIPTION_CHANNELS.COMMENT_LEFT_MODERATION_QUEUE,
payload: {
queue: GQLMODERATION_QUEUE.UNMODERATED,
commentID: comment.id,
storyID: comment.storyID,
},
});
}
}
+1
View File
@@ -0,0 +1 @@
export * from "./comments";
+8 -6
View File
@@ -3,7 +3,7 @@ import sinon from "sinon";
import { Config } from "coral-server/config";
import {
createJWTSigningConfig,
extractJWTFromRequest,
extractTokenFromRequest,
} from "coral-server/services/jwt";
import { Request } from "coral-server/types/express";
@@ -16,22 +16,22 @@ describe("extractJWTFromRequest", () => {
url: "",
};
expect(extractJWTFromRequest((req as any) as Request)).toEqual("token");
expect(extractTokenFromRequest((req as any) as Request)).toEqual("token");
delete req.headers.authorization;
expect(extractJWTFromRequest((req as any) as Request)).toEqual(null);
expect(extractTokenFromRequest((req as any) as Request)).toEqual(null);
});
it("extracts the token from query string", () => {
const req = {
url: "",
};
expect(extractJWTFromRequest((req as any) as Request)).toEqual(null);
expect(extractTokenFromRequest((req as any) as Request)).toEqual(null);
req.url = "https://coral.coralproject.net/api?accessToken=token";
expect(extractJWTFromRequest((req as any) as Request)).toEqual("token");
expect(extractTokenFromRequest((req as any) as Request)).toEqual("token");
});
it("does not extract the token from query string when it's disabled", () => {
@@ -39,7 +39,9 @@ describe("extractJWTFromRequest", () => {
url: "https://coral.coralproject.net/api?accessToken=token",
};
expect(extractJWTFromRequest((req as any) as Request, true)).toEqual(null);
expect(extractTokenFromRequest((req as any) as Request, true)).toEqual(
null
);
});
});
+3 -2
View File
@@ -10,6 +10,7 @@ import { AuthenticationError, TokenInvalidError } from "coral-server/errors";
import { Tenant } from "coral-server/models/tenant";
import { User } from "coral-server/models/user";
import { Request } from "coral-server/types/express";
import { IncomingMessage } from "http";
/**
* The following Claim Names are registered in the IANA "JSON Web Token
@@ -244,8 +245,8 @@ export async function signString<T extends {}>(
* @param req the request to extract the JWT from
* @param excludeQuery when true, does not pull from the query params
*/
export function extractJWTFromRequest(
req: Request,
export function extractTokenFromRequest(
req: Request | IncomingMessage,
excludeQuery: boolean = false
) {
const options: BearerOptions = {
+44
View File
@@ -0,0 +1,44 @@
import { Counter, Histogram } from "prom-client";
export interface Metrics {
executedGraphQueriesTotalCounter: Counter;
graphQLExecutionTimingsHistogram: Histogram;
httpRequestsTotal: Counter;
httpRequestDurationMilliseconds: Histogram;
}
export function createMetrics(): Metrics {
// Configure the metrics handlers.
const executedGraphQueriesTotalCounter = new Counter({
name: "coral_executed_graph_queries_total",
help: "number of GraphQL queries executed",
labelNames: ["operation_type", "operation_name"],
});
const graphQLExecutionTimingsHistogram = new Histogram({
name: "coral_executed_graph_queries_timings",
help: "timings for execution times of GraphQL operations",
buckets: [0.1, 5, 15, 50, 100, 500],
labelNames: ["operation_type", "operation_name"],
});
const httpRequestsTotal = new Counter({
name: "http_requests_total",
help: "Total number of HTTP requests made.",
labelNames: ["code", "method"],
});
const httpRequestDurationMilliseconds = new Histogram({
name: "http_request_duration_milliseconds",
help: "Histogram of latencies for HTTP requests.",
buckets: [0.1, 5, 15, 50, 100, 500],
labelNames: ["method", "handler"],
});
return {
executedGraphQueriesTotalCounter,
graphQLExecutionTimingsHistogram,
httpRequestsTotal,
httpRequestDurationMilliseconds,
};
}