[CORL-836] Create activeStories endpoint (#2787)

* Create activeStories GraphQL query endpoint

Set lastCommentedAt on stories when they are
commented upon.
Use lastCommentedAt to retrieve the activeStories.
Create a migration to partial index
lastCommentedAt on stories to make retrieval fast.

CORL-836

* fix: adjusted query to use index, more @auth directives

Co-authored-by: Wyatt Johnson <accounts+github@wyattjoh.ca>

src/core/server/graph/loaders/Comments.ts

@@ -3,19 +3,6 @@ import { defaultTo, isNil, omitBy } from "lodash";
 import { DateTime } from "luxon";
 
 import Context from "coral-server/graph/context";
-import {
-  CommentToParentsArgs,
-  CommentToRepliesArgs,
-  GQLActionPresence,
-  GQLCOMMENT_SORT,
-  GQLTAG,
-  GQLUSER_ROLE,
-  QueryToCommentsArgs,
-  StoryToCommentsArgs,
-  UserToAllCommentsArgs,
-  UserToCommentsArgs,
-  UserToRejectedCommentsArgs,
-} from "coral-server/graph/schema/__generated__/types";
 import { retrieveManyUserActionPresence } from "coral-server/models/action/comment";
 import {
   Comment,
@@ -36,6 +23,20 @@ import { Connection } from "coral-server/models/helpers";
 import { retrieveSharedModerationQueueQueuesCounts } from "coral-server/models/story/counts/shared";
 import { User } from "coral-server/models/user";
 
+import {
+  CommentToParentsArgs,
+  CommentToRepliesArgs,
+  GQLActionPresence,
+  GQLCOMMENT_SORT,
+  GQLTAG,
+  GQLUSER_ROLE,
+  QueryToCommentsArgs,
+  StoryToCommentsArgs,
+  UserToAllCommentsArgs,
+  UserToCommentsArgs,
+  UserToRejectedCommentsArgs,
+} from "coral-server/graph/schema/__generated__/types";
+
 import { SingletonResolver } from "./util";
 
 const tagFilter = (tag?: GQLTAG): CommentConnectionInput["filter"] => {

src/core/server/graph/loaders/Stories.ts

@@ -2,12 +2,9 @@ import DataLoader from "dataloader";
 import { defaultTo } from "lodash";
 
 import GraphContext from "coral-server/graph/context";
-import {
-  GQLSTORY_STATUS,
-  QueryToStoriesArgs,
-} from "coral-server/graph/schema/__generated__/types";
 import { Connection } from "coral-server/models/helpers";
 import {
+  retrieveActiveStories,
   retrieveManyStories,
   retrieveStoryConnection,
   Story,
@@ -21,6 +18,11 @@ import {
 } from "coral-server/services/stories";
 import { scraper } from "coral-server/services/stories/scraper";
 
+import {
+  GQLSTORY_STATUS,
+  QueryToStoriesArgs,
+} from "coral-server/graph/schema/__generated__/types";
+
 import { createManyBatchLoadFn } from "./util";
 
 const statusFilter = (
@@ -127,4 +129,6 @@ export default (ctx: GraphContext) => ({
       cache: !ctx.disableCaching,
     }
   ),
+  activeStories: (limit: number) =>
+    retrieveActiveStories(ctx.mongo, ctx.tenant.id, limit),
 });

src/core/server/graph/resolvers/Query.ts

@@ -21,4 +21,6 @@ export const Query: Required<GQLQueryTypeResolver<void>> = {
   debugScrapeStoryMetadata: (source, { url }, ctx) =>
     ctx.loaders.Stories.debugScrapeMetadata.load(url),
   moderationQueues: moderationQueuesResolver,
+  activeStories: (source, { limit = 10 }, ctx) =>
+    ctx.loaders.Stories.activeStories(limit),
 };

src/core/server/graph/schema/schema.graphql

@@ -769,10 +769,11 @@ type Auth {
   authentication solutions.
   """
   integrations: AuthIntegrations!
+
   """
   sessionDuration determines the duration in seconds for which an access token is valid
   """
-  sessionDuration: Int!
+  sessionDuration: Int! @auth(roles: [ADMIN])
 }
 
 ################################################################################
@@ -1597,6 +1598,7 @@ type PremodStatusHistory {
   active when true, indicates that the given user is premodded.
   """
   active: Boolean!
+
   """
   createdBy is the user that flagged the commenter as pre-mod
   """
@@ -1815,6 +1817,7 @@ type User {
   badges are user display badges
   """
   badges: [String!]
+
   """
   email is the current email address for the User.
   """
@@ -1953,17 +1956,32 @@ type User {
   of their account data.
   """
   lastDownloadedAt: Time
+    @auth(
+      userIDField: "id"
+      roles: [ADMIN, MODERATOR]
+      permit: [SUSPENDED, BANNED, PENDING_DELETION]
+    )
 
   """
   scheduledDeletionDate is the time when the User is
   scheduled to be deleted.
   """
   scheduledDeletionDate: Time
+    @auth(
+      userIDField: "id"
+      roles: [ADMIN, MODERATOR]
+      permit: [SUSPENDED, BANNED, PENDING_DELETION]
+    )
 
   """
   deletedAt is the time when the User was deleted.
   """
   deletedAt: Time
+    @auth(
+      userIDField: "id"
+      roles: [ADMIN, MODERATOR]
+      permit: [SUSPENDED, BANNED, PENDING_DELETION]
+    )
 }
 
 """
@@ -2583,7 +2601,7 @@ type Story {
   scrapedAt is the Time that the Story had it's metadata scraped at. If the time
   is null, the Story has not been scraped yet.
   """
-  scrapedAt: Time
+  scrapedAt: Time @auth(roles: [ADMIN, MODERATOR])
 
   """
   featuredComments are the Comments with the FEATURED tag on the Story.
@@ -2641,6 +2659,11 @@ type Story {
   settings.
   """
   settings: StorySettings!
+
+  """
+  lastCommentedAt is the last time someone commented on this story.
+  """
+  lastCommentedAt: Time @auth(roles: [ADMIN])
 }
 
 """
@@ -2780,6 +2803,14 @@ type Query {
   """
   moderationQueues(storyID: ID): ModerationQueues!
     @auth(roles: [ADMIN, MODERATOR])
+
+  """
+  activeStories is the list of most recently commented on stories identified
+  by their `lastCommentedAt` field
+  """
+  activeStories(limit: Int = 10 @constraint(max: 25)): [Story!]!
+    @auth(roles: [ADMIN])
+    @rate(max: 2, seconds: 1)
 }
 
 ################################################################################

src/core/server/models/comment/comment.ts

@@ -6,12 +6,6 @@ import uuid from "uuid";
 import { Omit, Sub } from "coral-common/types";
 import { dotize } from "coral-common/utils/dotize";
 import { CommentNotFoundError } from "coral-server/errors";
-import {
-  GQLCOMMENT_SORT,
-  GQLCOMMENT_STATUS,
-  GQLCommentTagCounts,
-  GQLTAG,
-} from "coral-server/graph/schema/__generated__/types";
 import logger from "coral-server/logger";
 import {
   EncodedCommentActionCounts,
@@ -31,6 +25,13 @@ import {
 import { TenantResource } from "coral-server/models/tenant";
 import { comments as collection } from "coral-server/services/mongodb/collections";
 
+import {
+  GQLCOMMENT_SORT,
+  GQLCOMMENT_STATUS,
+  GQLCommentTagCounts,
+  GQLTAG,
+} from "coral-server/graph/schema/__generated__/types";
+
 import { PUBLISHED_STATUSES } from "./constants";
 import {
   CommentStatusCounts,

src/core/server/models/story/index.ts

@@ -7,10 +7,6 @@ import {
   DuplicateStoryIDError,
   DuplicateStoryURLError,
 } from "coral-server/errors";
-import {
-  GQLStoryMetadata,
-  GQLStorySettings,
-} from "coral-server/graph/schema/__generated__/types";
 import {
   Connection,
   ConnectionInput,
@@ -21,6 +17,11 @@ import { GlobalModerationSettings } from "coral-server/models/settings";
 import { TenantResource } from "coral-server/models/tenant";
 import { stories as collection } from "coral-server/services/mongodb/collections";
 
+import {
+  GQLStoryMetadata,
+  GQLStorySettings,
+} from "coral-server/graph/schema/__generated__/types";
+
 import { createEmptyCommentStatusCounts } from "../comment/helpers";
 import {
   createEmptyCommentModerationQueueCounts,
@@ -75,6 +76,11 @@ export interface Story extends TenantResource {
    * createdAt is the date that the Story was added to the Coral database.
    */
   createdAt: Date;
+
+  /**
+   * lastCommentedAt is the last time someone commented on this story.
+   */
+  lastCommentedAt?: Date;
 }
 
 export interface UpsertStoryInput {
@@ -455,3 +461,43 @@ async function retrieveConnection(
   // Return a connection.
   return resolveConnection(query, input, story => story.createdAt);
 }
+
+export async function retrieveActiveStories(
+  mongo: Db,
+  tenantID: string,
+  limit: number
+) {
+  const stories = await collection(mongo)
+    .find({
+      tenantID,
+      // We limit this query to stories that have the following field. This
+      // allows us to use the index.
+      lastCommentedAt: {
+        $exists: true,
+      },
+    })
+    .sort({ lastCommentedAt: -1 })
+    .limit(limit)
+    .toArray();
+
+  return stories;
+}
+
+export async function updateStoryLastCommentedAt(
+  mongo: Db,
+  tenantID: string,
+  storyID: string,
+  now: Date
+) {
+  await collection(mongo).updateOne(
+    {
+      tenantID,
+      id: storyID,
+    },
+    {
+      $set: {
+        lastCommentedAt: now,
+      },
+    }
+  );
+}

src/core/server/services/migrate/migrations/1578604997397_story_add_last_commented_at.ts

@@ -0,0 +1,21 @@
+import { Db } from "mongodb";
+
+import Migration from "coral-server/services/migrate/migration";
+import collections from "coral-server/services/mongodb/collections";
+
+import { createIndexFactory } from "../indexing";
+
+export default class extends Migration {
+  public async indexes(mongo: Db) {
+    const createIndex = createIndexFactory(collections.stories(mongo));
+
+    // Create a partial sparse index on lastCommentedAt:
+    //   tenantID: 1 <- ASC tenantIDs
+    //   lastCommentedAt: -1 <- DESC dates (more recent to latest)
+    //   ^ explained here: https://docs.mongodb.com/manual/core/index-compound/#sort-order
+    await createIndex(
+      { tenantID: 1, lastCommentedAt: -1 },
+      { partialFilterExpression: { lastCommentedAt: { $exists: true } } }
+    );
+  }
+}

src/core/server/stacks/createComment.ts

@@ -25,7 +25,10 @@ import {
   hasAncestors,
   hasPublishedStatus,
 } from "coral-server/models/comment/helpers";
-import { retrieveStory } from "coral-server/models/story";
+import {
+  retrieveStory,
+  updateStoryLastCommentedAt,
+} from "coral-server/models/story";
 import { Tenant } from "coral-server/models/tenant";
 import { User } from "coral-server/models/user";
 import {
@@ -174,7 +177,10 @@ export default async function create(
     now
   );
 
-  await updateUserLastCommentID(redis, tenant, author, comment.id);
+  await Promise.all([
+    updateUserLastCommentID(redis, tenant, author, comment.id),
+    updateStoryLastCommentedAt(mongo, tenant.id, story.id, now),
+  ]);
 
   // Pull the revision out.
   const revision = getLatestRevision(comment);