[CORL-1002] Queue Improvements (#2931)

* fix: improved count handler for stories not found

* feat: removed performance-now

* feat: cleaned up processors, exposed counts

* feat: increased verb of schd jobs

* fix: removed dead code

src/core/server/graph/context.ts

@@ -12,8 +12,10 @@ import { PersistedQuery } from "coral-server/models/queries";
 import { Tenant } from "coral-server/models/tenant";
 import { User } from "coral-server/models/user";
 import { MailerQueue } from "coral-server/queue/tasks/mailer";
+import { NotifierQueue } from "coral-server/queue/tasks/notifier";
 import { RejectorQueue } from "coral-server/queue/tasks/rejector";
 import { ScraperQueue } from "coral-server/queue/tasks/scraper";
+import { WebhookQueue } from "coral-server/queue/tasks/webhook";
 import { I18n } from "coral-server/services/i18n";
 import { JWTSigningConfig } from "coral-server/services/jwt";
 import { AugmentedRedis } from "coral-server/services/redis";
@@ -38,11 +40,13 @@ export interface GraphContextOptions {
   config: Config;
   i18n: I18n;
   mailerQueue: MailerQueue;
+  rejectorQueue: RejectorQueue;
+  scraperQueue: ScraperQueue;
+  webhookQueue: WebhookQueue;
+  notifierQueue: NotifierQueue;
   mongo: Db;
   pubsub: RedisPubSub;
   redis: AugmentedRedis;
-  rejectorQueue: RejectorQueue;
-  scraperQueue: ScraperQueue;
   tenant: Tenant;
   tenantCache: TenantCache;
   broker: CoralEventListenerBroker;
@@ -58,13 +62,15 @@ export default class GraphContext {
   public readonly loaders: ReturnType<typeof loaders>;
   public readonly logger: Logger;
   public readonly mailerQueue: MailerQueue;
+  public readonly rejectorQueue: RejectorQueue;
+  public readonly scraperQueue: ScraperQueue;
+  public readonly webhookQueue: WebhookQueue;
+  public readonly notifierQueue: NotifierQueue;
   public readonly mongo: Db;
   public readonly mutators: ReturnType<typeof mutators>;
   public readonly now: Date;
   public readonly pubsub: RedisPubSub;
   public readonly redis: AugmentedRedis;
-  public readonly rejectorQueue: RejectorQueue;
-  public readonly scraperQueue: ScraperQueue;
   public readonly tenant: Tenant;
   public readonly tenantCache: TenantCache;
 
@@ -98,6 +104,8 @@ export default class GraphContext {
     this.scraperQueue = options.scraperQueue;
     this.mailerQueue = options.mailerQueue;
     this.rejectorQueue = options.rejectorQueue;
+    this.notifierQueue = options.notifierQueue;
+    this.webhookQueue = options.webhookQueue;
     this.signingConfig = options.signingConfig;
     this.clientID = options.clientID;
 

src/core/server/graph/extensions/LoggerExtension.ts

@@ -4,9 +4,9 @@ import {
   GraphQLExtension,
   GraphQLResponse,
 } from "graphql-extensions";
-import now from "performance-now";
 
 import GraphContext from "coral-server/graph/context";
+import { createTimer } from "coral-server/helpers";
 import logger from "coral-server/logger";
 
 import { getOperationMetadata, getPersistedQueryMetadata } from "./helpers";
@@ -84,17 +84,14 @@ export class LoggerExtension implements GraphQLExtension<GraphContext> {
     if (o.executionArgs.contextValue) {
       // Grab the start time so we can calculate the time it takes to execute
       // the graph query.
-      const startTime = now();
+      const timer = createTimer();
       return () => {
-        // Compute the end time.
-        const responseTime = Math.round(now() - startTime);
-
         // Log out the details of the request.
         logQuery(
           o.executionArgs.contextValue,
           o.executionArgs.document,
           undefined,
-          responseTime
+          timer()
         );
       };
     }

src/core/server/graph/extensions/MetricsExtension.ts

@@ -1,8 +1,9 @@
-import GraphContext from "coral-server/graph/context";
-import { Metrics } from "coral-server/services/metrics";
 import { ExecutionArgs } from "graphql";
 import { EndHandler, GraphQLExtension } from "graphql-extensions";
-import now from "performance-now";
+
+import GraphContext from "coral-server/graph/context";
+import { createTimer } from "coral-server/helpers";
+import { Metrics } from "coral-server/services/metrics";
 
 import { getOperationMetadata } from "./helpers";
 
@@ -16,10 +17,10 @@ export class MetricsExtension implements GraphQLExtension<GraphContext> {
     if (o.executionArgs.contextValue) {
       // Grab the start time so we can calculate the time it takes to execute
       // the graph query.
-      const startTime = now();
+      const timer = createTimer();
       return () => {
         // Compute the end time.
-        const responseTime = Math.round(now() - startTime);
+        const responseTime = timer();
 
         // Get the request metadata.
         const { operation, operationName } = getOperationMetadata(

src/core/server/graph/resolvers/Query.ts

@@ -28,4 +28,5 @@ export const Query: Required<GQLQueryTypeResolver<void>> = {
   sites: (source, args, ctx) => ctx.loaders.Sites.connection(args),
   site: (source, { id }, ctx) => (id ? ctx.loaders.Sites.site.load(id) : null),
   webhookEndpoint: (source, { id }, ctx) => getWebhookEndpoint(ctx.tenant, id),
+  queues: () => ({}),
 };

src/core/server/graph/resolvers/Queue.ts

@@ -0,0 +1,12 @@
+import {
+  GQLQueueCounts,
+  GQLQueueTypeResolver,
+} from "../schema/__generated__/types";
+
+export interface QueueInput {
+  counts(): Promise<GQLQueueCounts>;
+}
+
+export const Queue: Required<GQLQueueTypeResolver<QueueInput>> = {
+  counts: t => t.counts(),
+};

src/core/server/graph/resolvers/Queues.ts

@@ -0,0 +1,23 @@
+import { GQLQueuesTypeResolver } from "../schema/__generated__/types";
+
+import GraphContext from "../context";
+import { QueueInput } from "./Queue";
+
+/**
+ * get produces a resolver that maps the context to a QueueInput.
+ *
+ * @param fn the function to map the ctx to the queue.
+ */
+const get = (fn: (ctx: GraphContext) => QueueInput) => (
+  parent: any,
+  args: any,
+  ctx: GraphContext
+) => fn(ctx);
+
+export const Queues: Required<GQLQueuesTypeResolver> = {
+  mailer: get(ctx => ctx.mailerQueue),
+  scraper: get(ctx => ctx.scraperQueue),
+  notifier: get(ctx => ctx.notifierQueue),
+  webhook: get(ctx => ctx.webhookQueue),
+  rejector: get(ctx => ctx.rejectorQueue),
+};

src/core/server/graph/resolvers/index.ts

@@ -35,6 +35,8 @@ import { PremodStatus } from "./PremodStatus";
 import { PremodStatusHistory } from "./PremodStatusHistory";
 import { Profile } from "./Profile";
 import { Query } from "./Query";
+import { Queue } from "./Queue";
+import { Queues } from "./Queues";
 import { RecentCommentHistory } from "./RecentCommentHistory";
 import { RejectCommentPayload } from "./RejectCommentPayload";
 import { Secret } from "./Secret";
@@ -99,6 +101,8 @@ const Resolvers: GQLResolver = {
   Tag,
   Time,
   User,
+  Queue,
+  Queues,
   UsernameHistory,
   UsernameStatus,
   UserStatus,

src/core/server/graph/schema/schema.graphql

@@ -2997,6 +2997,62 @@ type StoriesConnection {
   pageInfo: PageInfo!
 }
 
+################################################################################
+## Queue
+################################################################################
+
+type QueueCounts {
+  """
+  waiting is the number of jobs that are in line to be processed.
+  """
+  waiting: Int!
+
+  """
+  active is the number of jobs that are being activly processed.
+  """
+  active: Int!
+
+  """
+  delayed is the number of jobs that have been delayed due to a failure and are
+  waiting for a backoff.
+  """
+  delayed: Int!
+}
+
+type Queue {
+  """
+  counts is the current counts associated with the Queue.
+  """
+  counts: QueueCounts!
+}
+
+type Queues {
+  """
+  mailer is the Queue associated with the Mailer queue.
+  """
+  mailer: Queue!
+
+  """
+  scraper is the Queue associated with the Scraper queue.
+  """
+  scraper: Queue!
+
+  """
+  notifier is the Queue associated with the Notifier queue.
+  """
+  notifier: Queue!
+
+  """
+  webhook is the Queue associated with the Webhook queue.
+  """
+  webhook: Queue!
+
+  """
+  rejector is the Queue associated with the Rejector queue.
+  """
+  rejector: Queue!
+}
+
 ################################################################################
 ## Query
 ################################################################################
@@ -3119,6 +3175,11 @@ type Query {
   webhookEndpint will return a specific WebhookEndpoint if it exists.
   """
   webhookEndpoint(id: ID!): WebhookEndpoint @auth(roles: [ADMIN])
+
+  """
+  queues returns information on queues used in Coral to manage
+  """
+  queues: Queues! @auth(roles: [ADMIN])
 }
 
 ################################################################################