From d4b8e5ef700a2b06174a72cca8660f1a413edb84 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Thu, 9 May 2019 22:26:24 +0000 Subject: [PATCH] [CORL-281] Metrics (#2298) * feat: iunitial metrics implementation * fix: graphql endpoint was throwing errors. * feat: add metrics env variables to readme --- README.md | 5 + package-lock.json | 142 ++++++++++++++---- package.json | 6 + src/core/index.ts | 4 +- src/core/server/app/helpers/entrypoints.ts | 5 +- src/core/server/app/index.ts | 5 + src/core/server/app/middleware/basicAuth.ts | 32 ++++ src/core/server/app/middleware/error.ts | 16 +- .../server/app/middleware/graphql/index.ts | 22 +++ src/core/server/app/middleware/logging.ts | 15 +- src/core/server/app/middleware/metrics.ts | 38 +++++ src/core/server/app/router/index.ts | 22 +++ src/core/server/config.ts | 22 +++ src/core/server/errors/index.ts | 19 ++- .../common/extensions/LoggerExtension.ts | 31 +--- .../common/extensions/MetricsExtension.ts | 51 +++++++ .../server/graph/common/extensions/helpers.ts | 22 +++ .../server/graph/common/extensions/index.ts | 1 + src/core/server/index.ts | 76 +++++++++- src/index.ts | 16 +- src/types/tsscmp.d.ts | 3 + 21 files changed, 461 insertions(+), 92 deletions(-) create mode 100644 src/core/server/app/middleware/basicAuth.ts create mode 100644 src/core/server/app/middleware/metrics.ts create mode 100644 src/core/server/graph/common/extensions/MetricsExtension.ts create mode 100644 src/core/server/graph/common/extensions/helpers.ts create mode 100644 src/types/tsscmp.d.ts diff --git a/README.md b/README.md index c6641d1de..6bd37db05 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,11 @@ the variables in a `.env` file in the root of the project in a simple `os.cpus().length`) - `DEV_PORT` - The port where the Webpack Development server is running on. (Default `8080`) +- `METRICS_USERNAME` - The username for _Basic Authentication_ at the `/metrics` and `/cluster_metrics` + endpoint. +- `METRICS_PASSWORD` - The password for _Basic Authentication_ at the `/metrics` and `/cluster_metrics` + endpoint. +- `CLUSTER_METRICS_PORT` - If `CONCURRENCY` is more than `1`, the metrics are provided at this port under `/cluster_metrics`. (Default `3001`) ## License diff --git a/package-lock.json b/package-lock.json index 8f1d08abe..d101c5608 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3342,6 +3342,15 @@ "integrity": "sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA==", "dev": true }, + "@types/basic-auth": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/basic-auth/-/basic-auth-1.1.2.tgz", + "integrity": "sha512-NzkkcC+gkkILWaBi3+/z/3do6Ybk6TWeTqV5zCVXmG2KaBoT5YqlJvfqP44HCyDA+Cu58pp7uKAxy/G58se/TA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/bcryptjs": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/bcryptjs/-/bcryptjs-2.4.1.tgz", @@ -3807,6 +3816,15 @@ "@types/node": "*" } }, + "@types/on-finished": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@types/on-finished/-/on-finished-2.3.1.tgz", + "integrity": "sha512-mzVYaYcFs5Jd2n/O6uYIRUsFRR1cHyZLRvkLCU0E7+G5WhY0qBDAR5fUCeZbvecYOSh9ikhlesyi2UfI8B9ckQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/passport": { "version": "0.4.6", "resolved": "https://registry.npmjs.org/@types/passport/-/passport-0.4.6.tgz", @@ -6704,6 +6722,21 @@ "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==", "dev": true }, + "basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -6747,6 +6780,11 @@ "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.11.0.tgz", "integrity": "sha1-RqoXUftqL5PuXmibsQh9SxTGwgU=" }, + "bintrees": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", + "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" + }, "bluebird": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", @@ -7723,11 +7761,6 @@ } } }, - "chownr": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz", - "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE=" - }, "chrome-trace-event": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.0.tgz", @@ -14682,8 +14715,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "1.2.4", @@ -15870,10 +15902,8 @@ "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.1.tgz", "integrity": "sha512-O+v1r9yN4tOsvl90p5HAP4AEqbYhx4036AGMm075fH9F8Qwi3oJ+v4u50FkT/KkvywNGtwkk0zRI+8eYm1X/xg==", "requires": { - "chownr": "^1.0.1", "fs-minipass": "^1.2.5", "minipass": "^2.2.4", - "minizlib": "^1.1.0", "mkdirp": "^0.5.0", "safe-buffer": "^5.1.1", "yallist": "^3.0.2" @@ -21862,14 +21892,6 @@ } } }, - "minizlib": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.1.0.tgz", - "integrity": "sha512-4T6Ur/GctZ27nHfpt9THOdRZNgyJ9FZchYO1ceg5S8Q3DNLCKYy44nCZzgCJgcvx2UM8czmqak5BCxJMrq37lA==", - "requires": { - "minipass": "^2.2.1" - } - }, "mississippi": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", @@ -27156,6 +27178,14 @@ "log-update": "^2.3.0" } }, + "prom-client": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.3.0.tgz", + "integrity": "sha512-OqSf5WOvpGZXkfqPXUHNHpjrbEE/q8jxjktO0i7zg1cnULAtf0ET67/J5R4e4iA4MZx2260tzTzSFSWgMdTZmQ==", + "requires": { + "tdigest": "^0.1.1" + } + }, "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", @@ -31501,21 +31531,56 @@ "dev": true }, "tar": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.1.tgz", - "integrity": "sha512-O+v1r9yN4tOsvl90p5HAP4AEqbYhx4036AGMm075fH9F8Qwi3oJ+v4u50FkT/KkvywNGtwkk0zRI+8eYm1X/xg==", + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", "dev": true, "optional": true, "requires": { - "chownr": "^1.0.1", + "chownr": "^1.1.1", "fs-minipass": "^1.2.5", - "minipass": "^2.2.4", - "minizlib": "^1.1.0", + "minipass": "^2.3.4", + "minizlib": "^1.1.1", "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.1", + "safe-buffer": "^5.1.2", "yallist": "^3.0.2" }, "dependencies": { + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", + "dev": true, + "optional": true + }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "dev": true, + "optional": true, + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", + "dev": true, + "optional": true, + "requires": { + "minipass": "^2.2.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "optional": true + }, "yallist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", @@ -31525,6 +31590,14 @@ } } }, + "tdigest": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", + "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "requires": { + "bintrees": "1.0.1" + } + }, "terser": { "version": "3.17.0", "resolved": "https://registry.npmjs.org/terser/-/terser-3.17.0.tgz", @@ -32694,6 +32767,11 @@ } } }, + "tsscmp": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", + "integrity": "sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==" + }, "tsutils": { "version": "2.29.0", "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", @@ -34927,6 +35005,18 @@ "requires": { "string-width": "^1.0.2 || 2" } + }, + "wrappy": { + "version": "1.0.2", + "bundled": true, + "dev": true, + "optional": true + }, + "yallist": { + "version": "3.0.3", + "bundled": true, + "dev": true, + "optional": true } } }, @@ -34968,9 +35058,7 @@ "yallist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", - "dev": true, - "optional": true + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" } } }, diff --git a/package.json b/package.json index 60c1746eb..e4a5367af 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@coralproject/bunyan-prettystream": "^0.1.4", "akismet-api": "^4.2.0", "apollo-server-express": "^2.1.0", + "basic-auth": "^2.0.1", "bcryptjs": "^2.4.3", "bull": "^3.8.1", "bunyan": "^1.8.12", @@ -105,6 +106,7 @@ "node-fetch": "^2.2.0", "nodemailer": "^4.6.7", "nunjucks": "^3.1.3", + "on-finished": "^2.3.0", "passport": "^0.4.0", "passport-facebook": "^2.1.1", "passport-google-oauth2": "^0.1.6", @@ -113,6 +115,7 @@ "passport-strategy": "^1.0.0", "performance-now": "^2.1.0", "permit": "^0.2.4", + "prom-client": "^11.3.0", "querystringify": "^2.1.0", "react-relay-network-modern": "^2.4.0", "source-map-support": "^0.5.12", @@ -121,6 +124,7 @@ "throng": "^4.0.0", "tlds": "^1.203.1", "ts-node-dev": "^1.0.0-pre.37", + "tsscmp": "^1.0.6", "uuid": "^3.3.2", "verror": "^1.10.0" }, @@ -133,6 +137,7 @@ "@babel/preset-react": "^7.0.0", "@coralproject/rte": "^0.10.13", "@intervolga/optimize-cssnano-plugin": "^1.0.6", + "@types/basic-auth": "^1.1.2", "@types/bcryptjs": "^2.4.1", "@types/bull": "^3.5.12", "@types/bunyan": "^1.8.4", @@ -172,6 +177,7 @@ "@types/node-fetch": "^2.3.3", "@types/nodemailer": "^4.6.2", "@types/nunjucks": "^3.1.1", + "@types/on-finished": "^2.3.1", "@types/passport": "^0.4.6", "@types/passport-facebook": "^2.1.8", "@types/passport-local": "^1.0.33", diff --git a/src/core/index.ts b/src/core/index.ts index 097e43ba1..4104469fe 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -5,9 +5,7 @@ import Server, { ServerOptions } from "./server"; * * @param options ServerOptions that will be used to configure Talk. */ -export default async function createTalk( - options: ServerOptions = {} -): Promise { +export default function createTalk(options: ServerOptions = {}): Server { // Create the server with the provided options. return new Server(options); } diff --git a/src/core/server/app/helpers/entrypoints.ts b/src/core/server/app/helpers/entrypoints.ts index e2fd22622..d8e792fef 100644 --- a/src/core/server/app/helpers/entrypoints.ts +++ b/src/core/server/app/helpers/entrypoints.ts @@ -104,7 +104,10 @@ export default class Entrypoints { // Create and return the entrypoints. return new Entrypoints(manifest); } catch (err) { - logger.error({ err }, "could not load the manifest"); + logger.error( + { err }, + "could not load the manifest, maybe you need to run `npm run build`" + ); return null; } } diff --git a/src/core/server/app/index.ts b/src/core/server/app/index.ts index af202c963..1d3f02a53 100644 --- a/src/core/server/app/index.ts +++ b/src/core/server/app/index.ts @@ -20,6 +20,7 @@ import { AugmentedRedis } from "talk-server/services/redis"; import TenantCache from "talk-server/services/tenant/cache"; import { accessLogger, errorLogger } from "./middleware/logging"; +import { metricsRecorder } from "./middleware/metrics"; import serveStatic from "./middleware/serveStatic"; import { createRouter } from "./router"; @@ -34,6 +35,7 @@ export interface AppOptions { schema: GraphQLSchema; signingConfig: JWTSigningConfig; tenantCache: TenantCache; + metrics: boolean; } /** @@ -49,6 +51,9 @@ export async function createApp(options: AppOptions): Promise { // Logging parent.use(accessLogger); + // Capturing metrics. + parent.use(metricsRecorder()); + // Create some services for the router. const passport = createPassport(options); diff --git a/src/core/server/app/middleware/basicAuth.ts b/src/core/server/app/middleware/basicAuth.ts new file mode 100644 index 000000000..ca3278c3e --- /dev/null +++ b/src/core/server/app/middleware/basicAuth.ts @@ -0,0 +1,32 @@ +import auth from "basic-auth"; +import compare from "tsscmp"; + +import { RequestHandler } from "talk-server/types/express"; + +export const basicAuth = ( + username: string, + password: string +): RequestHandler => { + function check(name: string, pass: string) { + let valid = true; + + // Simple method to prevent short-circuit and use timing-safe compare. + valid = compare(name, username) && valid; + valid = compare(pass, password) && valid; + + return valid; + } + + return (req, res, next) => { + // Pull the credentials out of the request. + const credentials = auth(req); + + // Check credentials + if (credentials && check(credentials.name, credentials.pass)) { + return next(); + } + + res.setHeader("WWW-Authenticate", `Basic realm="${req.originalUrl}"`); + res.status(401).send("Access denied"); + }; +}; diff --git a/src/core/server/app/middleware/error.ts b/src/core/server/app/middleware/error.ts index 2f74c060d..daa501237 100644 --- a/src/core/server/app/middleware/error.ts +++ b/src/core/server/app/middleware/error.ts @@ -1,5 +1,6 @@ import { ErrorRequestHandler } from "express"; +import { FluentBundle } from "fluent/compat"; import { InternalError, TalkError } from "talk-server/errors"; import { I18n } from "talk-server/services/i18n"; import { Request } from "talk-server/types/express"; @@ -22,11 +23,14 @@ const wrapError = (err: Error) => * @param bundles the translation bundles * @param tenant the optional tenant to use when selecting the language */ -const serializeError = (err: TalkError, req: Request, bundles: I18n) => { +const serializeError = (err: TalkError, req: Request, bundles?: I18n) => { // Get the translation bundle. - let bundle = bundles.getDefaultBundle(); - if (req.talk && req.talk.tenant) { - bundle = bundles.getBundle(req.talk.tenant.locale); + let bundle: FluentBundle | null = null; + if (bundles) { + bundle = bundles.getDefaultBundle(); + if (req.talk && req.talk.tenant) { + bundle = bundles.getBundle(req.talk.tenant.locale); + } } return { @@ -34,7 +38,7 @@ const serializeError = (err: TalkError, req: Request, bundles: I18n) => { }; }; -export const JSONErrorHandler = (bundles: I18n): ErrorRequestHandler => ( +export const JSONErrorHandler = (bundles?: I18n): ErrorRequestHandler => ( err, req, res, @@ -47,7 +51,7 @@ export const JSONErrorHandler = (bundles: I18n): ErrorRequestHandler => ( res.status(err.status).json(serializeError(err, req, bundles)); }; -export const HTMLErrorHandler = (bundles: I18n): ErrorRequestHandler => ( +export const HTMLErrorHandler = (bundles?: I18n): ErrorRequestHandler => ( err, req, res, diff --git a/src/core/server/app/middleware/graphql/index.ts b/src/core/server/app/middleware/graphql/index.ts index 77ab7c972..163d69c74 100644 --- a/src/core/server/app/middleware/graphql/index.ts +++ b/src/core/server/app/middleware/graphql/index.ts @@ -1,6 +1,7 @@ import { GraphQLOptions } from "apollo-server-express"; import { Handler } from "express"; import { FieldDefinitionNode, GraphQLError, ValidationContext } from "graphql"; +import { Counter, Histogram } from "prom-client"; // TODO: when https://github.com/apollographql/apollo-server/pull/1907 is merged, update this import path import { @@ -13,6 +14,7 @@ import { Config } from "talk-server/config"; import { ErrorWrappingExtension, LoggerExtension, + MetricsExtension, } from "talk-server/graph/common/extensions"; export * from "./batch"; @@ -42,6 +44,20 @@ export const graphqlMiddleware = ( config: Config, requestOptions: ExpressGraphQLOptionsFunction ): Handler => { + // Configure the metrics handlers. + const executedGraphQueriesTotalCounter = new Counter({ + name: "talk_executed_graph_queries_total", + help: "number of GraphQL queries executed", + labelNames: ["operation_type", "operation_name"], + }); + + const graphQLExecutionTimingsHistogram = new Histogram({ + name: "talk_executed_graph_queries_timings", + help: "timings for execution times of GraphQL operations", + buckets: [0.1, 5, 15, 50, 100, 500], + labelNames: ["operation_type", "operation_name"], + }); + // Create a new baseOptions that will be merged into the new options. const baseOptions: Omit = { // Disable the debug mode, as we already add in our logging function. @@ -50,6 +66,12 @@ export const graphqlMiddleware = ( extensions: [ () => new ErrorWrappingExtension(), () => new LoggerExtension(), + () => + // Pass the metrics to the extension so it can increment. + new MetricsExtension({ + executedGraphQueriesTotalCounter, + graphQLExecutionTimingsHistogram, + }), ], }; diff --git a/src/core/server/app/middleware/logging.ts b/src/core/server/app/middleware/logging.ts index 72e0a563c..4e537d212 100644 --- a/src/core/server/app/middleware/logging.ts +++ b/src/core/server/app/middleware/logging.ts @@ -1,26 +1,19 @@ import { ErrorRequestHandler, RequestHandler } from "express"; +import onFinished from "on-finished"; import now from "performance-now"; import logger from "talk-server/logger"; export const accessLogger: RequestHandler = (req, res, next) => { const startTime = now(); - const end = res.end; - res.end = (chunk: any, encodingOrCb?: any, cb?: any) => { + + onFinished(res, () => { // Compute the end time. const responseTime = Math.round(now() - startTime); // Get some extra goodies from the request. const userAgent = req.get("User-Agent"); - // Reattach the old end, and finish. - res.end = end; - if (typeof encodingOrCb === "function") { - res.end(chunk, encodingOrCb); - } else { - res.end(chunk, encodingOrCb, cb); - } - // Log this out. logger.info( { @@ -33,7 +26,7 @@ export const accessLogger: RequestHandler = (req, res, next) => { }, "http request" ); - }; + }); next(); }; diff --git a/src/core/server/app/middleware/metrics.ts b/src/core/server/app/middleware/metrics.ts new file mode 100644 index 000000000..c2fc491ce --- /dev/null +++ b/src/core/server/app/middleware/metrics.ts @@ -0,0 +1,38 @@ +import { RequestHandler } from "express"; +import onFinished from "on-finished"; +import now from "performance-now"; +import { Counter, Histogram } from "prom-client"; + +export const metricsRecorder = (): RequestHandler => { + const httpRequestsTotal = new Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests made.", + labelNames: ["code", "method"], + }); + + const httpRequestDurationMilliseconds = new Histogram({ + name: "http_request_duration_milliseconds", + help: "Histogram of latencies for HTTP requests.", + buckets: [0.1, 5, 15, 50, 100, 500], + labelNames: ["method", "handler"], + }); + + return (req, res, next) => { + const startTime = now(); + + onFinished(res, () => { + // Compute the end time. + const responseTime = Math.round(now() - startTime); + + // Increment the request counter. + httpRequestsTotal.labels(`${res.statusCode}`, req.method).inc(); + + // Add the request duration. + httpRequestDurationMilliseconds + .labels(req.method, req.baseUrl + req.path) + .observe(responseTime); + }); + + next(); + }; +}; diff --git a/src/core/server/app/router/index.ts b/src/core/server/app/router/index.ts index 315470524..c128e8a55 100644 --- a/src/core/server/app/router/index.ts +++ b/src/core/server/app/router/index.ts @@ -1,5 +1,6 @@ import express, { Router } from "express"; import path from "path"; +import { register } from "prom-client"; import { AppOptions } from "talk-server/app"; import { noCacheMiddleware } from "talk-server/app/middleware/cacheHeaders"; @@ -11,6 +12,7 @@ import { RouterOptions } from "talk-server/app/router/types"; import logger from "talk-server/logger"; import Entrypoints from "../helpers/entrypoints"; +import { basicAuth } from "../middleware/basicAuth"; import { createAPIRouter } from "./api"; import { createClientTargetRouter } from "./client"; @@ -131,6 +133,26 @@ export function createRouter(app: AppOptions, options: RouterOptions) { ); } + if (app.metrics) { + // Add basic auth if provided. + const username = app.config.get("metrics_username"); + const password = app.config.get("metrics_password"); + if (username && password) { + router.use("/metrics", basicAuth(username, password)); + logger.info("adding authentication to metrics endpoint"); + } else { + logger.info( + "not adding authentication to metrics endpoint, credentials not provided" + ); + } + + router.get("/metrics", noCacheMiddleware, (req, res) => { + res.set("Content-Type", register.contentType); + res.end(register.metrics()); + }); + logger.info({ path: "/metrics" }, "mounting metrics path on app"); + } + return router; } diff --git a/src/core/server/config.ts b/src/core/server/config.ts index 9652bfcb2..9e29bc6c3 100644 --- a/src/core/server/config.ts +++ b/src/core/server/config.ts @@ -81,6 +81,28 @@ const config = convict({ env: "PORT", arg: "port", }, + cluster_metrics_port: { + doc: "The port to bind for cluster metrics.", + format: "port", + default: 3001, + env: "CLUSTER_METRICS_PORT", + arg: "clusterMetricsPort", + }, + metrics_username: { + doc: "The username to use to authenticate to the metrics endpoint.", + format: "String", + default: "", + env: "METRICS_USERNAME", + arg: "metricsUsername", + }, + metrics_password: { + doc: "The password to use to authenticate to the metrics endpoint.", + format: "String", + default: "", + env: "METRICS_PASSWORD", + arg: "metricsPassword", + sensitive: true, + }, dev_port: { doc: "The port to bind for the Webpack Dev Server.", format: "port", diff --git a/src/core/server/errors/index.ts b/src/core/server/errors/index.ts index 0f1b1e422..98100a310 100644 --- a/src/core/server/errors/index.ts +++ b/src/core/server/errors/index.ts @@ -172,13 +172,18 @@ export class TalkError extends VError { this.param = param; } - public serializeExtensions(bundle: FluentBundle): TalkErrorExtensions { - const message = translate( - bundle, - this.code, - ERROR_TRANSLATIONS[this.code], - this.context.pub - ); + public serializeExtensions(bundle: FluentBundle | null): TalkErrorExtensions { + let message: string; + if (bundle) { + message = translate( + bundle, + this.code, + ERROR_TRANSLATIONS[this.code], + this.context.pub + ); + } else { + message = this.code; + } return { id: this.id, diff --git a/src/core/server/graph/common/extensions/LoggerExtension.ts b/src/core/server/graph/common/extensions/LoggerExtension.ts index 87b7b2bd3..22b772c91 100644 --- a/src/core/server/graph/common/extensions/LoggerExtension.ts +++ b/src/core/server/graph/common/extensions/LoggerExtension.ts @@ -1,9 +1,4 @@ -import { - DocumentNode, - ExecutionArgs, - GraphQLError, - OperationDefinitionNode, -} from "graphql"; +import { ExecutionArgs, GraphQLError } from "graphql"; import { EndHandler, GraphQLExtension, @@ -12,33 +7,13 @@ import { import now from "performance-now"; import CommonContext from "talk-server/graph/common/context"; +import { getOperationMetadata } from "./helpers"; export function logError(ctx: CommonContext, err: GraphQLError) { ctx.logger.error({ err }, "graphql query error"); } export class LoggerExtension implements GraphQLExtension { - private getOperationMetadata(doc: DocumentNode) { - if (doc.kind === "Document") { - const operationDefinition = doc.definitions.find( - ({ kind }) => kind === "OperationDefinition" - ) as OperationDefinitionNode | undefined; - if (operationDefinition) { - let operationName: string | undefined; - if (operationDefinition.name) { - operationName = operationDefinition.name.value; - } - - return { - operationName, - operation: operationDefinition.operation, - }; - } - } - - return {}; - } - public executionDidStart(o: { executionArgs: ExecutionArgs; }): EndHandler | void { @@ -55,7 +30,7 @@ export class LoggerExtension implements GraphQLExtension { o.executionArgs.contextValue.logger.debug( { responseTime, - ...this.getOperationMetadata(o.executionArgs.document), + ...getOperationMetadata(o.executionArgs.document), }, "graphql query" ); diff --git a/src/core/server/graph/common/extensions/MetricsExtension.ts b/src/core/server/graph/common/extensions/MetricsExtension.ts new file mode 100644 index 000000000..f101d0a17 --- /dev/null +++ b/src/core/server/graph/common/extensions/MetricsExtension.ts @@ -0,0 +1,51 @@ +import { ExecutionArgs } from "graphql"; +import { EndHandler, GraphQLExtension } from "graphql-extensions"; +import now from "performance-now"; +import { Counter, Histogram } from "prom-client"; + +import CommonContext from "talk-server/graph/common/context"; +import { getOperationMetadata } from "./helpers"; + +export interface MetricsExtensionOptions { + executedGraphQueriesTotalCounter: Counter; + graphQLExecutionTimingsHistogram: Histogram; +} + +export class MetricsExtension implements GraphQLExtension { + private options: MetricsExtensionOptions; + + constructor(options: MetricsExtensionOptions) { + this.options = options; + } + + public executionDidStart(o: { + executionArgs: ExecutionArgs; + }): EndHandler | void { + // Only try to log things if the context is provided. + if (o.executionArgs.contextValue) { + // Grab the start time so we can calculate the time it takes to execute + // the graph query. + const startTime = now(); + return () => { + // Compute the end time. + const responseTime = Math.round(now() - startTime); + + // Get the request metadata. + const { operation, operationName } = getOperationMetadata( + o.executionArgs.document + ); + + if (operation && operationName) { + // Increment the graph query value, tagging with the name of the query. + this.options.executedGraphQueriesTotalCounter + .labels(operation, operationName) + .inc(); + + this.options.graphQLExecutionTimingsHistogram + .labels(operation, operationName) + .observe(responseTime); + } + }; + } + } +} diff --git a/src/core/server/graph/common/extensions/helpers.ts b/src/core/server/graph/common/extensions/helpers.ts new file mode 100644 index 000000000..3fa651a60 --- /dev/null +++ b/src/core/server/graph/common/extensions/helpers.ts @@ -0,0 +1,22 @@ +import { DocumentNode, OperationDefinitionNode } from "graphql"; + +export function getOperationMetadata(doc: DocumentNode) { + if (doc.kind === "Document") { + const operationDefinition = doc.definitions.find( + ({ kind }) => kind === "OperationDefinition" + ) as OperationDefinitionNode | undefined; + if (operationDefinition) { + let operationName: string | undefined; + if (operationDefinition.name) { + operationName = operationDefinition.name.value; + } + + return { + operationName, + operation: operationDefinition.operation, + }; + } + } + + return {}; +} diff --git a/src/core/server/graph/common/extensions/index.ts b/src/core/server/graph/common/extensions/index.ts index 3b901c243..465746062 100644 --- a/src/core/server/graph/common/extensions/index.ts +++ b/src/core/server/graph/common/extensions/index.ts @@ -1,2 +1,3 @@ export * from "./ErrorWrappingExtension"; export * from "./LoggerExtension"; +export * from "./MetricsExtension"; diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 904158858..b1e3a7424 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -1,7 +1,9 @@ +import cluster from "cluster"; import express, { Express } from "express"; import { GraphQLSchema } from "graphql"; import http from "http"; import { Db } from "mongodb"; +import { AggregatorRegistry, collectDefaultMetrics } from "prom-client"; import { LanguageCode } from "talk-common/helpers/i18n/locales"; import { createApp, listenAndServe } from "talk-server/app"; @@ -19,6 +21,11 @@ import { createRedisClient, } from "talk-server/services/redis"; import TenantCache from "talk-server/services/tenant/cache"; +import { basicAuth } from "./app/middleware/basicAuth"; +import { noCacheMiddleware } from "./app/middleware/cacheHeaders"; +import { JSONErrorHandler } from "./app/middleware/error"; +import { accessLogger, errorLogger } from "./app/middleware/logging"; +import { notFoundMiddleware } from "./app/middleware/notFound"; export interface ServerOptions { /** @@ -27,6 +34,10 @@ export interface ServerOptions { config?: Config; } +export interface ServerStartOptions { + parent?: Express; +} + /** * Server provides an interface to create, start, and manage a Talk Server. */ @@ -122,6 +133,9 @@ class Server { tenantCache: this.tenantCache, i18n: this.i18n, }); + + // Setup the metrics collectors. + collectDefaultMetrics({ timeout: 5000 }); } /** @@ -146,6 +160,62 @@ class Server { // Launch all of the job processors. this.tasks.mailer.process(); this.tasks.scraper.process(); + + // If we are running in concurrency mode, and we are the master, we should + // setup the aggregator for the cluster metrics. + if (cluster.isMaster && this.config.get("concurrency") > 1) { + // Create the aggregator registry for metrics. + const aggregatorRegistry = new AggregatorRegistry(); + + // Setup the cluster metrics server. + const metricsServer = express(); + + // Setup access logger. + metricsServer.use(accessLogger); + + // Add basic auth if provided. + const username = this.config.get("metrics_username"); + const password = this.config.get("metrics_password"); + if (username && password) { + metricsServer.use("/cluster_metrics", basicAuth(username, password)); + logger.info("adding authentication to metrics endpoint"); + } else { + logger.info( + "not adding authentication to metrics endpoint, credentials not provided" + ); + } + + // Cluster metrics will be served on /cluster_metrics. + metricsServer.get( + "/cluster_metrics", + noCacheMiddleware, + (req, res, next) => { + aggregatorRegistry.clusterMetrics((err, metrics) => { + if (err) { + return next(err); + } + + res.set("Content-Type", aggregatorRegistry.contentType); + res.send(metrics); + }); + } + ); + + // Error handling. + metricsServer.use(notFoundMiddleware); + metricsServer.use(errorLogger); + metricsServer.use(JSONErrorHandler()); + + const port = this.config.get("cluster_metrics_port"); + + // Star the server listening for cluster metrics. + await listenAndServe(metricsServer, port); + + logger.info( + { port, path: "/cluster_metrics" }, + "now listening for cluster metrics" + ); + } } /** @@ -154,7 +224,7 @@ class Server { * * @param parent the optional express application to bind the server to. */ - public async start(parent?: Express) { + public async start({ parent }: ServerStartOptions) { // Guard against not being connected. if (!this.connected) { throw new Error("server has not connected yet"); @@ -168,6 +238,9 @@ class Server { // Create the signing config. const signingConfig = createJWTSigningConfig(this.config); + // Only enable the metrics server if concurrency is set to 1. + const metrics = this.config.get("concurrency") === 1; + // Create the Talk App, branching off from the parent app. const app: Express = await createApp({ parent, @@ -180,6 +253,7 @@ class Server { i18n: this.i18n, mailerQueue: this.tasks.mailer, scraperQueue: this.tasks.scraper, + metrics, }); // Start the application and store the resulting http.Server. The server diff --git a/src/index.ts b/src/index.ts index 832f308ed..d0b959621 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,15 +36,15 @@ import Server from "./core/server"; import logger from "./core/server/logger"; // Create the app that will serve as the mounting point for the Talk Server. -const app = express(); +const parent = express(); // worker will start the worker process. async function worker(server: Server) { try { - logger.debug("started server worker"); - // Start the server. - await server.start(app); + await server.start({ parent }); + + logger.debug("started server worker"); } catch (err) { logger.error({ err }, "can not start server in worker mode"); throw err; @@ -57,10 +57,10 @@ async function master(server: Server) { logger.debug({ workerCount }, "spawning workers to handle traffic"); try { - logger.debug("started server master"); - // Process jobs. await server.process(); + + logger.debug("started server master"); } catch (err) { logger.error({ err }, "can not start server in master mode"); throw err; @@ -73,7 +73,7 @@ async function bootstrap() { logger.debug("starting bootstrap"); // Create the server instance. - const server = await createTalk(); + const server = createTalk(); // Determine the number of workers. const workerCount = server.config.get("concurrency"); @@ -91,7 +91,7 @@ async function bootstrap() { await server.process(); // Start the server. - await server.start(app); + await server.start({ parent }); } else { // Launch the server start within throng. throng({ diff --git a/src/types/tsscmp.d.ts b/src/types/tsscmp.d.ts new file mode 100644 index 000000000..c6ebd0cda --- /dev/null +++ b/src/types/tsscmp.d.ts @@ -0,0 +1,3 @@ +declare module "tsscmp" { + export default function tsscmp(expect: string, got: string): boolean; +}