[next] Story Mutations (#2055)

*  feat: added mutations for removing + scraping stories

- added mutation for removing stories
- added mutation for scraping stories
- renamed all delete* to remove* to avoid reserved keyword collision

* fix: linting

* feat: implemented createStory

* feat: added support for createStory

* feat: improved Story deletion

- Deletes comments on the deleted Story
- Deletes actions of comments that were deleted via new
  `root_item_id` parameter for Comment Action's
- Added logging around critical components of the remove
  process

* feat: implemented mergeStory

- Added new merge function for root actions
- Added new merge function for story comments
- Added new merge function for comment status counts
- Added new remove stories function

* fix: fixed story creation method

* fix: added action count merging

* fix: removed outdated file

* fix: removed queue that got duplicated via rebase

* fix: prevent error when action counts are empty

src/core/client/stream/mutations/RemoveCommentReactionMutation.ts

@@ -6,12 +6,12 @@ import {
   createMutationContainer,
 } from "talk-framework/lib/relay";
 
-import { DeleteCommentReactionMutation as MutationTypes } from "talk-stream/__generated__/DeleteCommentReactionMutation.graphql";
-import { CreateCommentReactionInput } from "./CreateCommentReactionMutation";
+import { RemoveCommentReactionMutation as MutationTypes } from "talk-stream/__generated__/RemoveCommentReactionMutation.graphql";
+import { CreateCommentReactionInput } from "talk-stream/mutations/CreateCommentReactionMutation";
 
 const mutation = graphql`
-  mutation DeleteCommentReactionMutation($input: CreateCommentReactionInput!) {
-    deleteCommentReaction(input: $input) {
+  mutation RemoveCommentReactionMutation($input: CreateCommentReactionInput!) {
+    removeCommentReaction(input: $input) {
       comment {
         ...ReactionButtonContainer_comment
       }
@@ -36,7 +36,7 @@ function commit(environment: Environment, input: CreateCommentReactionInput) {
       },
     },
     optimisticResponse: {
-      deleteCommentReaction: {
+      removeCommentReaction: {
         comment: {
           id: input.commentID,
           myActionPresence: {
@@ -54,11 +54,11 @@ function commit(environment: Environment, input: CreateCommentReactionInput) {
   });
 }
 
-export const withDeleteCommentReactionMutation = createMutationContainer(
-  "deleteCommentReaction",
+export const withRemoveCommentReactionMutation = createMutationContainer(
+  "removeCommentReaction",
   commit
 );
 
-export type DeleteCommentReactionMutation = (
+export type RemoveCommentReactionMutation = (
   input: CreateCommentReactionInput
-) => Promise<MutationTypes["response"]["deleteCommentReaction"]>;
+) => Promise<MutationTypes["response"]["removeCommentReaction"]>;

src/core/client/stream/mutations/index.ts

@@ -32,6 +32,6 @@ export {
   CreateCommentReactionInput,
 } from "./CreateCommentReactionMutation";
 export {
-  withDeleteCommentReactionMutation,
-  DeleteCommentReactionMutation,
-} from "./DeleteCommentReactionMutation";
+  withRemoveCommentReactionMutation,
+  RemoveCommentReactionMutation,
+} from "./RemoveCommentReactionMutation";

src/core/client/stream/tabs/comments/containers/ReactionButtonContainer.tsx

@@ -7,9 +7,9 @@ import { ReactionButtonContainer_settings as SettingsData } from "talk-stream/__
 
 import {
   CreateCommentReactionMutation,
-  DeleteCommentReactionMutation,
+  RemoveCommentReactionMutation,
   withCreateCommentReactionMutation,
-  withDeleteCommentReactionMutation,
+  withRemoveCommentReactionMutation,
 } from "talk-stream/mutations";
 import ReactionButton from "talk-stream/tabs/comments/components/ReactionButton";
 
@@ -20,7 +20,7 @@ import {
 
 interface ReactionButtonContainerProps {
   createCommentReaction: CreateCommentReactionMutation;
-  deleteCommentReaction: DeleteCommentReactionMutation;
+  removeCommentReaction: RemoveCommentReactionMutation;
   comment: CommentData;
   settings: SettingsData;
   me: MeData | null;
@@ -41,13 +41,13 @@ class ReactionButtonContainer extends React.Component<
       commentID: this.props.comment.id,
     };
 
-    const { createCommentReaction, deleteCommentReaction } = this.props;
+    const { createCommentReaction, removeCommentReaction } = this.props;
     const reacted =
       this.props.comment.myActionPresence &&
       this.props.comment.myActionPresence.reaction;
 
     return reacted
-      ? deleteCommentReaction(input)
+      ? removeCommentReaction(input)
       : createCommentReaction(input);
   };
   public render() {
@@ -79,7 +79,7 @@ class ReactionButtonContainer extends React.Component<
 }
 
 export default withShowAuthPopupMutation(
-  withDeleteCommentReactionMutation(
+  withRemoveCommentReactionMutation(
     withCreateCommentReactionMutation(
       withFragmentContainer<ReactionButtonContainerProps>({
         me: graphql`

src/core/server/app/index.ts

@@ -14,8 +14,8 @@ import { notFoundMiddleware } from "talk-server/app/middleware/notFound";
 import { createPassport } from "talk-server/app/middleware/passport";
 import { handleSubscriptions } from "talk-server/graph/common/subscriptions/middleware";
 import { Schemas } from "talk-server/graph/schemas";
+import { TaskQueue } from "talk-server/queue";
 import { JWTSigningConfig } from "talk-server/services/jwt";
-import { TaskQueue } from "talk-server/services/queue";
 import TenantCache from "talk-server/services/tenant/cache";
 
 import { accessLogger, errorLogger } from "./middleware/logging";

src/core/server/app/middleware/context/tenant.ts

@@ -4,7 +4,7 @@ import { Db } from "mongodb";
 
 import { Config } from "talk-common/config";
 import TenantContext from "talk-server/graph/tenant/context";
-import { TaskQueue } from "talk-server/services/queue";
+import { TaskQueue } from "talk-server/queue";
 import { Request } from "talk-server/types/express";
 
 export interface TenantContextMiddlewareOptions {

src/core/server/graph/tenant/context.ts

@@ -4,7 +4,7 @@ import { Db } from "mongodb";
 import CommonContext from "talk-server/graph/common/context";
 import { Tenant } from "talk-server/models/tenant";
 import { User } from "talk-server/models/user";
-import { TaskQueue } from "talk-server/services/queue";
+import { TaskQueue } from "talk-server/queue";
 import TenantCache from "talk-server/services/tenant/cache";
 import { Request } from "talk-server/types/express";
 

src/core/server/graph/tenant/loaders/stories.ts

@@ -7,8 +7,8 @@ import {
   retrieveManyStories,
   Story,
 } from "talk-server/models/story";
-import { scraper } from "talk-server/services/queue/tasks/scraper";
 import { findOrCreate } from "talk-server/services/stories";
+import { scraper } from "talk-server/services/stories/scraper";
 
 export default (ctx: TenantContext) => ({
   findOrCreate: (input: FindOrCreateStoryInput) =>

src/core/server/graph/tenant/mutators/comment.ts

@@ -4,19 +4,19 @@ import {
   GQLCreateCommentFlagInput,
   GQLCreateCommentInput,
   GQLCreateCommentReactionInput,
-  GQLDeleteCommentDontAgreeInput,
-  GQLDeleteCommentFlagInput,
-  GQLDeleteCommentReactionInput,
   GQLEditCommentInput,
+  GQLRemoveCommentDontAgreeInput,
+  GQLRemoveCommentFlagInput,
+  GQLRemoveCommentReactionInput,
 } from "talk-server/graph/tenant/schema/__generated__/types";
 import { create, edit } from "talk-server/services/comments";
 import {
   createDontAgree,
   createFlag,
   createReaction,
-  deleteDontAgree,
-  deleteFlag,
-  deleteReaction,
+  removeDontAgree,
+  removeFlag,
+  removeReaction,
 } from "talk-server/services/comments/actions";
 
 export default (ctx: TenantContext) => ({
@@ -48,16 +48,16 @@ export default (ctx: TenantContext) => ({
     createReaction(ctx.mongo, ctx.tenant, ctx.user!, {
       item_id: input.commentID,
     }),
-  deleteReaction: (input: GQLDeleteCommentReactionInput) =>
-    deleteReaction(ctx.mongo, ctx.tenant, ctx.user!, {
+  removeReaction: (input: GQLRemoveCommentReactionInput) =>
+    removeReaction(ctx.mongo, ctx.tenant, ctx.user!, {
       item_id: input.commentID,
     }),
   createDontAgree: (input: GQLCreateCommentDontAgreeInput) =>
     createDontAgree(ctx.mongo, ctx.tenant, ctx.user!, {
       item_id: input.commentID,
     }),
-  deleteDontAgree: (input: GQLDeleteCommentDontAgreeInput) =>
-    deleteDontAgree(ctx.mongo, ctx.tenant, ctx.user!, {
+  removeDontAgree: (input: GQLRemoveCommentDontAgreeInput) =>
+    removeDontAgree(ctx.mongo, ctx.tenant, ctx.user!, {
       item_id: input.commentID,
     }),
   createFlag: (input: GQLCreateCommentFlagInput) =>
@@ -65,8 +65,8 @@ export default (ctx: TenantContext) => ({
       item_id: input.commentID,
       reason: input.reason,
     }),
-  deleteFlag: (input: GQLDeleteCommentFlagInput) =>
-    deleteFlag(ctx.mongo, ctx.tenant, ctx.user!, {
+  removeFlag: (input: GQLRemoveCommentFlagInput) =>
+    removeFlag(ctx.mongo, ctx.tenant, ctx.user!, {
       item_id: input.commentID,
     }),
 });

src/core/server/graph/tenant/mutators/index.ts

@@ -2,8 +2,10 @@ import TenantContext from "talk-server/graph/tenant/context";
 
 import Comment from "./comment";
 import Settings from "./settings";
+import Story from "./story";
 
 export default (ctx: TenantContext) => ({
   Comment: Comment(ctx),
   Settings: Settings(ctx),
+  Story: Story(ctx),
 });

src/core/server/graph/tenant/mutators/settings.ts

@@ -3,15 +3,15 @@ import { isNull, omitBy } from "lodash";
 import TenantContext from "talk-server/graph/tenant/context";
 import {
   GQLCreateOIDCAuthIntegrationInput,
-  GQLDeleteOIDCAuthIntegrationInput,
+  GQLRemoveOIDCAuthIntegrationInput,
   GQLSettingsInput,
   GQLUpdateOIDCAuthIntegrationInput,
 } from "talk-server/graph/tenant/schema/__generated__/types";
 import { Tenant } from "talk-server/models/tenant";
 import {
   createOIDCAuthIntegration,
-  deleteOIDCAuthIntegration,
   regenerateSSOKey,
+  removeOIDCAuthIntegration,
   update,
   updateOIDCAuthIntegration,
 } from "talk-server/services/tenant";
@@ -38,6 +38,6 @@ export default ({ mongo, redis, tenantCache, tenant }: TenantContext) => ({
       input.id,
       input.configuration
     ),
-  deleteOIDCAuthIntegration: (input: GQLDeleteOIDCAuthIntegrationInput) =>
-    deleteOIDCAuthIntegration(mongo, redis, tenantCache, tenant, input.id),
+  removeOIDCAuthIntegration: (input: GQLRemoveOIDCAuthIntegrationInput) =>
+    removeOIDCAuthIntegration(mongo, redis, tenantCache, tenant, input.id),
 });

src/core/server/graph/tenant/mutators/story.ts

@@ -0,0 +1,32 @@
+import { isNull, omitBy } from "lodash";
+
+import TenantContext from "talk-server/graph/tenant/context";
+import {
+  GQLCreateStoryInput,
+  GQLMergeStoriesInput,
+  GQLRemoveStoryInput,
+  GQLScrapeStoryInput,
+  GQLUpdateStoryInput,
+} from "talk-server/graph/tenant/schema/__generated__/types";
+import { Story } from "talk-server/models/story";
+import { create, merge, remove, update } from "talk-server/services/stories";
+import { scrape } from "talk-server/services/stories/scraper";
+
+export default (ctx: TenantContext) => ({
+  create: async (input: GQLCreateStoryInput): Promise<Readonly<Story> | null> =>
+    create(
+      ctx.mongo,
+      ctx.tenant,
+      input.story.id,
+      input.story.url,
+      omitBy(input.story, isNull)
+    ),
+  update: async (input: GQLUpdateStoryInput): Promise<Readonly<Story> | null> =>
+    update(ctx.mongo, ctx.tenant, input.id, omitBy(input.story, isNull)),
+  merge: async (input: GQLMergeStoriesInput): Promise<Readonly<Story> | null> =>
+    merge(ctx.mongo, ctx.tenant, input.destinationID, input.sourceIDs),
+  remove: async (input: GQLRemoveStoryInput): Promise<Readonly<Story> | null> =>
+    remove(ctx.mongo, ctx.tenant, input.id, input.includeComments),
+  scrape: async (input: GQLScrapeStoryInput): Promise<Readonly<Story> | null> =>
+    scrape(ctx.mongo, ctx.tenant.id, input.id),
+});

src/core/server/graph/tenant/resolvers/mutation.ts

@@ -27,24 +27,24 @@ const Mutation: GQLMutationTypeResolver<void> = {
     comment: await ctx.mutators.Comment.createReaction(input),
     clientMutationId: input.clientMutationId,
   }),
-  deleteCommentReaction: async (source, { input }, ctx) => ({
-    comment: await ctx.mutators.Comment.deleteReaction(input),
+  removeCommentReaction: async (source, { input }, ctx) => ({
+    comment: await ctx.mutators.Comment.removeReaction(input),
     clientMutationId: input.clientMutationId,
   }),
   createCommentDontAgree: async (source, { input }, ctx) => ({
     comment: await ctx.mutators.Comment.createDontAgree(input),
     clientMutationId: input.clientMutationId,
   }),
-  deleteCommentDontAgree: async (source, { input }, ctx) => ({
-    comment: await ctx.mutators.Comment.deleteDontAgree(input),
+  removeCommentDontAgree: async (source, { input }, ctx) => ({
+    comment: await ctx.mutators.Comment.removeDontAgree(input),
     clientMutationId: input.clientMutationId,
   }),
   createCommentFlag: async (source, { input }, ctx) => ({
     comment: await ctx.mutators.Comment.createFlag(input),
     clientMutationId: input.clientMutationId,
   }),
-  deleteCommentFlag: async (source, { input }, ctx) => ({
-    comment: await ctx.mutators.Comment.deleteFlag(input),
+  removeCommentFlag: async (source, { input }, ctx) => ({
+    comment: await ctx.mutators.Comment.removeFlag(input),
     clientMutationId: input.clientMutationId,
   }),
   regenerateSSOKey: async (source, { input }, ctx) => ({
@@ -75,8 +75,8 @@ const Mutation: GQLMutationTypeResolver<void> = {
       clientMutationId: input.clientMutationId,
     };
   },
-  deleteOIDCAuthIntegration: async (source, { input }, ctx) => {
-    const result = await ctx.mutators.Settings.deleteOIDCAuthIntegration(input);
+  removeOIDCAuthIntegration: async (source, { input }, ctx) => {
+    const result = await ctx.mutators.Settings.removeOIDCAuthIntegration(input);
     if (!result) {
       return { clientMutationId: input.clientMutationId };
     }
@@ -87,6 +87,26 @@ const Mutation: GQLMutationTypeResolver<void> = {
       clientMutationId: input.clientMutationId,
     };
   },
+  createStory: async (source, { input }, ctx) => ({
+    story: await ctx.mutators.Story.create(input),
+    clientMutationId: input.clientMutationId,
+  }),
+  updateStory: async (source, { input }, ctx) => ({
+    story: await ctx.mutators.Story.update(input),
+    clientMutationId: input.clientMutationId,
+  }),
+  mergeStories: async (source, { input }, ctx) => ({
+    story: await ctx.mutators.Story.merge(input),
+    clientMutationId: input.clientMutationId,
+  }),
+  removeStory: async (source, { input }, ctx) => ({
+    story: await ctx.mutators.Story.remove(input),
+    clientMutationId: input.clientMutationId,
+  }),
+  scrapeStory: async (source, { input }, ctx) => ({
+    story: await ctx.mutators.Story.scrape(input),
+    clientMutationId: input.clientMutationId,
+  }),
 };
 
 export default Mutation;

src/core/server/graph/tenant/schema/schema.graphql

@@ -1052,7 +1052,7 @@ type Comment {
 
   """
   replyCount is the number of replies. Only direct replies to this Comment
-  are counted. Deleted comments are included in this count.
+  are counted. Removed comments are included in this count.
   """
   replyCount: Int!
 
@@ -1918,12 +1918,12 @@ type CreateCommentReactionPayload {
 }
 
 ##################
-## deleteCommentReaction
+## removeCommentReaction
 ##################
 
-input DeleteCommentReactionInput {
+input RemoveCommentReactionInput {
   """
-  commentID is the Comment's ID that we want to delete a Reaction on.
+  commentID is the Comment's ID that we want to remove a Reaction on.
   """
   commentID: ID!
 
@@ -1933,9 +1933,9 @@ input DeleteCommentReactionInput {
   clientMutationId: String!
 }
 
-type DeleteCommentReactionPayload {
+type RemoveCommentReactionPayload {
   """
-  comment is the Comment that the Reaction was deleted on.
+  comment is the Comment that the Reaction was removed on.
   """
   comment: Comment
 
@@ -1974,12 +1974,12 @@ type CreateCommentDontAgreePayload {
 }
 
 ##################
-## deleteCommentDontAgree
+## removeCommentDontAgree
 ##################
 
-input DeleteCommentDontAgreeInput {
+input RemoveCommentDontAgreeInput {
   """
-  commentID is the Comment's ID that we want to delete a DontAgree on.
+  commentID is the Comment's ID that we want to remove a DontAgree on.
   """
   commentID: ID!
 
@@ -1989,9 +1989,9 @@ input DeleteCommentDontAgreeInput {
   clientMutationId: String!
 }
 
-type DeleteCommentDontAgreePayload {
+type RemoveCommentDontAgreePayload {
   """
-  comment is the Comment that the DontAgree was deleted on.
+  comment is the Comment that the DontAgree was removed on.
   """
   comment: Comment
 
@@ -2035,12 +2035,12 @@ type CreateCommentFlagPayload {
 }
 
 ##################
-## deleteCommentFlag
+## removeCommentFlag
 ##################
 
-input DeleteCommentFlagInput {
+input RemoveCommentFlagInput {
   """
-  commentID is the Comment's ID that we want to delete a Flag on.
+  commentID is the Comment's ID that we want to remove a Flag on.
   """
   commentID: ID!
 
@@ -2050,9 +2050,9 @@ input DeleteCommentFlagInput {
   clientMutationId: String!
 }
 
-type DeleteCommentFlagPayload {
+type RemoveCommentFlagPayload {
   """
-  comment is the Comment that the Flag was deleted on.
+  comment is the Comment that the Flag was removed on.
   """
   comment: Comment
 
@@ -2278,10 +2278,10 @@ type UpdateOIDCAuthIntegrationPayload {
 }
 
 ##################
-# deleteOIDCAuthIntegration
+# removeOIDCAuthIntegration
 ##################
 
-input DeleteOIDCAuthIntegrationInput {
+input RemoveOIDCAuthIntegrationInput {
   """
   id is the ID of the specific OpenID Connect integration that we are deleting.
   """
@@ -2293,14 +2293,14 @@ input DeleteOIDCAuthIntegrationInput {
   clientMutationId: String!
 }
 
-type DeleteOIDCAuthIntegrationPayload {
+type RemoveOIDCAuthIntegrationPayload {
   """
-  integration is the OIDCAuthIntegration we just deleted.
+  integration is the OIDCAuthIntegration we just removed.
   """
   integration: OIDCAuthIntegration
 
   """
-  settings is the Settings that the OIDCAuthIntegration was deleted on with the
+  settings is the Settings that the OIDCAuthIntegration was removed on with the
   OIDCAuthIntegration removed from it.
   """
   settings: Settings
@@ -2311,6 +2311,252 @@ type DeleteOIDCAuthIntegrationPayload {
   clientMutationId: String!
 }
 
+##################
+## createStory
+##################
+
+"""
+StoryMetadataInput is the metadata for a given Story as provided via this API.
+"""
+input StoryMetadataInput {
+  """
+  title stores the title from the Story page.
+  """
+  title: String
+
+  """
+  author stores the author from the Story page.
+  """
+  author: String
+
+  """
+  description stores the description from the Story page.
+  """
+  description: String
+
+  """
+  image stores the image from the Story page.
+  """
+  image: String
+
+  """
+  publishedAt stores the publication date from the Story page.
+  """
+  publishedAt: Time
+
+  """
+  modifiedAt stores the modified date from the Story page.
+  """
+  modifiedAt: Time
+
+  """
+  section stores the section from the Story page.
+  """
+  section: String
+}
+
+"""
+CreateStory is the input required to create a Story.
+"""
+input CreateStory {
+  """
+  id is the identifier of the Story.
+  """
+  id: ID!
+
+  """
+  url is the url that the Story is located on.
+  """
+  url: String!
+
+  """
+  metadata is the set of information relating to this Story that would normally
+  be scraped, but can be provided here.
+  """
+  metadata: StoryMetadataInput
+}
+
+input CreateStoryInput {
+  """
+  story is the Story input needed to create a Story.
+  """
+  story: CreateStory!
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+type CreateStoryPayload {
+  """
+  story is the Story that was possibly created.
+  """
+  story: Story
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+##################
+## updateStory
+##################
+
+"""
+UpdateStory is the input required to update a Story.
+"""
+input UpdateStory {
+  """
+  url is the url that the Story is located on.
+  """
+  url: String
+
+  """
+  metadata is the set of information relating to this Story that would normally
+  be scraped, but can be provided here.
+  """
+  metadata: StoryMetadataInput
+
+  """
+  isClosed is true when the Story should be closed for commenting.
+  """
+  isClosed: Boolean
+}
+
+input UpdateStoryInput {
+  """
+  id is the identifier of the Story used either when the Story was created via
+  the API or from Talk when it was lazily created.
+  """
+  id: ID!
+
+  """
+  story contains the fields that should be updated. Any fields not specified
+  will not be changed.
+  """
+  story: UpdateStory!
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+type UpdateStoryPayload {
+  """
+  story is the Story that was possibly updated.
+  """
+  story: Story
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+##################
+## mergeStories
+##################
+
+input MergeStoriesInput {
+  """
+  sourceIDs are the list of Story ID's that should have their Comment's moved
+  onto the Story indicated by `destinationID`. The Stories indicated by the
+  `sourceIDs` field will be removed after the Comment's have been moved.
+  """
+  sourceIDs: [ID!]!
+
+  """
+  destinationID is the ID of the Story where all the other "source" Stories will
+  have their Comment's moved onto.
+  """
+  destinationID: ID!
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+type MergeStoriesPayload {
+  """
+  story is the Story that all the source stories were merged into.
+  """
+  story: Story
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+##################
+## removeStory
+##################
+
+input RemoveStoryInput {
+  """
+  id is the identifier of the Story used either when the Story was created via
+  the API or from Talk when it was lazily created.
+  """
+  id: ID!
+
+  """
+  includeComments when true will remove any Comment's that were left on the
+  Story. This option should be used rarely, instead preferring to updating the
+  Story URL and/or merging with the correct Story.
+  """
+  includeComments: Boolean = false
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+type RemoveStoryPayload {
+  """
+  story is the Story that was possibly removed.
+  """
+  story: Story
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+##################
+## scrapeStory
+##################
+
+input ScrapeStoryInput {
+  """
+  id is the identifier of the Story used either when the Story was created via
+  the API or from Talk when it was lazily created.
+  """
+  id: ID!
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
+type ScrapeStoryPayload {
+  """
+  story is the Story that was possibly scraped.
+  """
+  story: Story
+
+  """
+  clientMutationId is required for Relay support.
+  """
+  clientMutationId: String!
+}
+
 ##################
 ## Mutation
 ##################
@@ -2355,12 +2601,12 @@ type Mutation {
   ): UpdateOIDCAuthIntegrationPayload @auth(roles: [ADMIN])
 
   """
-  deleteOIDCAuthIntegration will delete the specified OpenID Connect auth
+  removeOIDCAuthIntegration will remove the specified OpenID Connect auth
   integration.
   """
-  deleteOIDCAuthIntegration(
-    input: DeleteOIDCAuthIntegrationInput!
-  ): DeleteOIDCAuthIntegrationPayload @auth(roles: [ADMIN])
+  removeOIDCAuthIntegration(
+    input: RemoveOIDCAuthIntegrationInput!
+  ): RemoveOIDCAuthIntegrationPayload @auth(roles: [ADMIN])
 
   """
   createCommentReaction will create a Reaction authored by the current logged in
@@ -2371,10 +2617,10 @@ type Mutation {
   ): CreateCommentReactionPayload @auth
 
   """
-  deleteCommentReaction will delete a Reaction authored by the current logged in
+  removeCommentReaction will remove a Reaction authored by the current logged in
   User on a Comment if it exists.
   """
-  deleteCommentReaction(
+  removeCommentReaction(
     input: CreateCommentReactionInput!
   ): CreateCommentReactionPayload @auth
 
@@ -2387,10 +2633,10 @@ type Mutation {
   ): CreateCommentDontAgreePayload @auth
 
   """
-  deleteCommentDontAgree will delete a DontAgree authored by the current logged in
+  removeCommentDontAgree will remove a DontAgree authored by the current logged in
   User on a Comment if it exists.
   """
-  deleteCommentDontAgree(
+  removeCommentDontAgree(
     input: CreateCommentDontAgreeInput!
   ): CreateCommentDontAgreePayload @auth
 
@@ -2402,11 +2648,42 @@ type Mutation {
     @auth
 
   """
-  deleteCommentFlag will create a Flag authored by the current logged in User on
+  removeCommentFlag will create a Flag authored by the current logged in User on
   a given Comment.
   """
-  deleteCommentFlag(input: DeleteCommentFlagInput!): DeleteCommentFlagPayload
+  removeCommentFlag(input: RemoveCommentFlagInput!): RemoveCommentFlagPayload
     @auth
+
+  """
+  createStory will create the provided Story.
+  """
+  createStory(input: CreateStoryInput!): CreateStoryPayload
+    @auth(roles: [ADMIN])
+
+  """
+  updateStory will update the given Story.
+  """
+  updateStory(input: UpdateStoryInput!): UpdateStoryPayload
+    @auth(roles: [ADMIN])
+
+  """
+  mergeStories will merge two stories together, merging their comment streams.
+  This operation is irreversible.
+  """
+  mergeStories(input: MergeStoriesInput!): MergeStoriesPayload
+    @auth(roles: [ADMIN])
+
+  """
+  removeStory will remove the given Story.
+  """
+  removeStory(input: RemoveStoryInput!): RemoveStoryPayload
+    @auth(roles: [ADMIN])
+
+  """
+  scrapeStory will scrape the given Story and update the scraped metadata.
+  """
+  scrapeStory(input: ScrapeStoryInput!): ScrapeStoryPayload
+    @auth(roles: [ADMIN, MODERATOR])
 }
 
 ################################################################################

src/core/server/index.ts

@@ -5,7 +5,7 @@ import config, { Config } from "talk-common/config";
 import getManagementSchema from "talk-server/graph/management/schema";
 import { Schemas } from "talk-server/graph/schemas";
 import getTenantSchema from "talk-server/graph/tenant/schema";
-import { createQueue } from "talk-server/services/queue";
+import { createQueue } from "talk-server/queue";
 import TenantCache from "talk-server/services/tenant/cache";
 
 import { createJWTSigningConfig } from "talk-server/services/jwt";

src/core/server/models/action.ts

@@ -56,13 +56,54 @@ export type FLAG_REASON =
   | GQLCOMMENT_FLAG_REASON;
 
 export interface Action extends TenantResource {
+  /**
+   * id is the identifier for this specific Action.
+   */
   readonly id: string;
+
+  /**
+   * action_type is the type of Action that this represents.
+   */
   action_type: ACTION_TYPE;
+
+  /**
+   * item_type enables polymorphic behavior be allowing multiple item types
+   * to be represented in a single collection.
+   */
   item_type: ACTION_ITEM_TYPE;
+
+  /**
+   * item_id is the ID of the specific item that this Action is associated with.
+   */
   item_id: string;
+
+  /**
+   * reason is the reason or secondary grouping identifier for why this
+   * particular action was left.
+   */
   reason?: FLAG_REASON;
+
+  /**
+   * root_item_id represents the identifier for the item's associated item. In
+   * the case of a REACTION left on a Comment, this ID would be the Stories ID.
+   * In the case of a FLAG left on a User, this ID would be null.
+   */
+  root_item_id?: string;
+
+  /**
+   * user_id is the ID of the User that left this Action. In the event that the
+   * Action was left by Talk, it will be null.
+   */
   user_id?: string;
+
+  /**
+   * created_at is the date that this particular Action was created at.
+   */
   created_at: Date;
+
+  /**
+   * metadata is arbitrary information stored for this Action.
+   */
   metadata?: Record<string, any>;
 }
 
@@ -248,36 +289,36 @@ export async function retrieveManyUserActionPresence(
     );
 }
 
-export type DeleteActionInput = Pick<
+export type RemoveActionInput = Pick<
   Action,
   "action_type" | "item_type" | "item_id" | "reason" | "user_id"
 >;
 
 /**
- * The result returned by `deleteAction`.
+ * The result returned by `removeAction`.
  */
-export interface DeletedActionResultObject {
+export interface RemovedActionResultObject {
   /**
    * action is the action that was deleted.
    */
   action?: Action;
 
   /**
-   * wasDeleted is true when the action that was supposed to be deleted was
+   * wasRemoved is true when the action that was supposed to be deleted was
    * actually deleted.
    */
-  wasDeleted: boolean;
+  wasRemoved: boolean;
 }
 
 /**
- * deleteAction will delete the action based on the form of the action rather
+ * removeAction will delete the action based on the form of the action rather
  * than a specific action by ID.
  */
-export async function deleteAction(
+export async function removeAction(
   mongo: Db,
   tenantID: string,
-  input: DeleteActionInput
-): Promise<DeletedActionResultObject> {
+  input: RemoveActionInput
+): Promise<RemovedActionResultObject> {
   // Extract the filter parameters.
   const filter: FilterQuery<Action> = {
     tenant_id: tenantID,
@@ -297,7 +338,7 @@ export async function deleteAction(
   const result = await collection(mongo).findOneAndDelete(filter);
   return {
     action: result.value,
-    wasDeleted: Boolean(result.ok && result.value),
+    wasRemoved: Boolean(result.ok && result.value),
   };
 }
 
@@ -455,6 +496,30 @@ function createEmptyActionCounts(): GQLActionCounts {
   };
 }
 
+export function mergeActionCounts(
+  actionCounts: EncodedActionCounts[]
+): EncodedActionCounts {
+  const mergedActionCounts: EncodedActionCounts = {};
+
+  for (const counts of actionCounts) {
+    for (const [key, count] of Object.entries(counts)) {
+      if (key in mergedActionCounts) {
+        mergedActionCounts[key] += count;
+      } else {
+        mergedActionCounts[key] += 1;
+      }
+    }
+  }
+
+  return mergedActionCounts;
+}
+
+export function countTotalActionCounts(
+  actionCounts: EncodedActionCounts
+): number {
+  return Object.values(actionCounts).reduce((total, count) => total + count, 0);
+}
+
 /**
  * decodeActionCounts will take the encoded action counts and decode them into
  * a useable format.
@@ -469,7 +534,7 @@ export function decodeActionCounts(
 
   // Loop over all the encoded action counts to extract each of the action
   // counts as they are encoded.
-  Object.entries(encodedActionCounts).map(([key, count]) => {
+  Object.entries(encodedActionCounts).forEach(([key, count]) => {
     // Pull out the action type and the reason from the key.
     const { actionType, reason } = decodeActionCountKey(key);
 
@@ -509,3 +574,43 @@ function incrementActionCounts(
 
   return actionCounts;
 }
+
+/**
+ * removeRootActions will remove all the Action's associated with a given root
+ * identifier.
+ */
+export async function removeRootActions(
+  mongo: Db,
+  tenantID: string,
+  rootItemID: string
+) {
+  return collection(mongo).deleteMany({
+    tenant_id: tenantID,
+    root_item_id: rootItemID,
+  });
+}
+
+/**
+ * mergeManyRootActions will update many Action `root_item_id'`s from one to
+ * another.
+ */
+export async function mergeManyRootActions(
+  mongo: Db,
+  tenantID: string,
+  newRootItemID: string,
+  oldRootItemIDs: string[]
+) {
+  return collection(mongo).updateMany(
+    {
+      tenant_id: tenantID,
+      root_item_id: {
+        $in: oldRootItemIDs,
+      },
+    },
+    {
+      $set: {
+        root_item_id: newRootItemID,
+      },
+    }
+  );
+}

src/core/server/models/comment.ts

@@ -550,3 +550,43 @@ export async function updateCommentActionCounts(
 
   return result.value;
 }
+
+/**
+ * removeStoryComments will remove all comments associated with a particular
+ * Story.
+ */
+export async function removeStoryComments(
+  mongo: Db,
+  tenantID: string,
+  storyID: string
+) {
+  // Delete all the comments written on a specific story.
+  return collection(mongo).deleteMany({
+    tenant_id: tenantID,
+    story_id: storyID,
+  });
+}
+
+/**
+ * mergeManyCommentStories will update many comment's storyID's.
+ */
+export async function mergeManyCommentStories(
+  mongo: Db,
+  tenantID: string,
+  newStoryID: string,
+  oldStoryIDs: string[]
+) {
+  return collection(mongo).updateMany(
+    {
+      tenant_id: tenantID,
+      story_id: {
+        $in: oldStoryIDs,
+      },
+    },
+    {
+      $set: {
+        story_id: newStoryID,
+      },
+    }
+  );
+}

src/core/server/models/story.ts

@@ -1,4 +1,4 @@
-import { Db } from "mongodb";
+import { Db, MongoError } from "mongodb";
 import uuid from "uuid";
 
 import { Omit } from "talk-common/types";
@@ -149,6 +149,37 @@ export async function updateCommentStatusCount(
   return result.value || null;
 }
 
+/**
+ * mergeCommentStatusCount will merge an array of commentStatusCount's into one.
+ */
+export function mergeCommentStatusCount(
+  commentStatusCounts: CommentStatusCounts[]
+): CommentStatusCounts {
+  const statusCounts = createEmptyCommentCounts();
+  for (const commentCounts of commentStatusCounts) {
+    for (const status in commentCounts) {
+      if (!commentCounts.hasOwnProperty(status)) {
+        continue;
+      }
+
+      // Because the CommentStatusCounts are not indexable, it should be accessed
+      // by walking the structure.
+      switch (status) {
+        case GQLCOMMENT_STATUS.ACCEPTED:
+        case GQLCOMMENT_STATUS.NONE:
+        case GQLCOMMENT_STATUS.PREMOD:
+        case GQLCOMMENT_STATUS.REJECTED:
+        case GQLCOMMENT_STATUS.SYSTEM_WITHHELD:
+          statusCounts[status] += commentCounts[status];
+          break;
+        default:
+          throw new Error("unrecognized status");
+      }
+    }
+  }
+  return statusCounts;
+}
+
 function createEmptyCommentCounts(): CommentStatusCounts {
   return {
     [GQLCOMMENT_STATUS.ACCEPTED]: 0,
@@ -191,6 +222,46 @@ export async function findOrCreateStory(
   return upsertStory(db, tenantID, { url });
 }
 
+export type CreateStoryInput = Partial<Pick<Story, "metadata">>;
+
+export async function createStory(
+  mongo: Db,
+  tenantID: string,
+  id: string,
+  url: string,
+  input: CreateStoryInput
+) {
+  const now = new Date();
+
+  // Create the story.
+  const story: Story = {
+    ...input,
+    id,
+    url,
+    tenant_id: tenantID,
+    created_at: now,
+    action_counts: {},
+    comment_counts: createEmptyCommentCounts(),
+  };
+
+  try {
+    // Insert the story into the database.
+    await collection(mongo).insertOne(story);
+  } catch (err) {
+    // Evaluate the error, if it is in regards to violating the unique index,
+    // then return a duplicate Story error.
+    if (err instanceof MongoError && err.code === 11000) {
+      // TODO: (wyattjoh) return better error
+      throw new Error("story with this url already exists");
+    }
+
+    throw err;
+  }
+
+  // Return the created story.
+  return story;
+}
+
 export async function retrieveStoryByURL(
   db: Db,
   tenantID: string,
@@ -235,7 +306,7 @@ export async function retrieveManyStoriesByURL(
 
 export type UpdateStoryInput = Omit<
   Partial<Story>,
-  "id" | "tenant_id" | "url" | "created_at"
+  "id" | "tenant_id" | "created_at"
 >;
 
 export async function updateStory(
@@ -253,15 +324,26 @@ export async function updateStory(
     },
   };
 
-  const result = await collection(db).findOneAndUpdate(
-    { id, tenant_id: tenantID },
-    update,
-    // False to return the updated document instead of the original
-    // document.
-    { returnOriginal: false }
-  );
+  try {
+    const result = await collection(db).findOneAndUpdate(
+      { id, tenant_id: tenantID },
+      update,
+      // False to return the updated document instead of the original
+      // document.
+      { returnOriginal: false }
+    );
 
-  return result.value || null;
+    return result.value || null;
+  } catch (err) {
+    // Evaluate the error, if it is in regards to violating the unique index,
+    // then return a duplicate Story error.
+    if (input.url && err instanceof MongoError && err.code === 11000) {
+      // TODO: (wyattjoh) return better error
+      throw new Error("story with this url already exists");
+    }
+
+    throw err;
+  }
 }
 
 /**
@@ -291,3 +373,58 @@ export async function updateStoryActionCounts(
 
   return result.value || null;
 }
+
+export async function removeStory(mongo: Db, tenantID: string, id: string) {
+  const result = await collection(mongo).findOneAndDelete({
+    id,
+    tenant_id: tenantID,
+  });
+
+  return result.value || null;
+}
+
+/**
+ * removeStories will remove the stories specified by the set of id's.
+ */
+export async function removeStories(
+  mongo: Db,
+  tenantID: string,
+  ids: string[]
+) {
+  return collection(mongo).deleteMany({
+    tenant_id: tenantID,
+    id: {
+      $in: ids,
+    },
+  });
+}
+
+/**
+ * calculateTotalCommentCount will compute the total amount of comments left on
+ * an Asset by parsing the `CommentStatusCounts`.
+ */
+export function calculateTotalCommentCount(
+  commentCounts: CommentStatusCounts
+): number {
+  let count = 0;
+  for (const status in commentCounts) {
+    if (!commentCounts.hasOwnProperty(status)) {
+      continue;
+    }
+
+    // Because the CommentStatusCounts are not indexable, it should be accessed
+    // by walking the structure.
+    switch (status) {
+      case GQLCOMMENT_STATUS.ACCEPTED:
+      case GQLCOMMENT_STATUS.NONE:
+      case GQLCOMMENT_STATUS.PREMOD:
+      case GQLCOMMENT_STATUS.REJECTED:
+      case GQLCOMMENT_STATUS.SYSTEM_WITHHELD:
+        count += commentCounts[status];
+        break;
+      default:
+        throw new Error("unrecognized status");
+    }
+  }
+  return count;
+}

src/core/server/models/tenant.ts

@@ -352,25 +352,25 @@ export async function updateTenantOIDCAuthIntegration(
   };
 }
 
-export interface DeleteTenantOIDCAuthIntegrationResultObject {
+export interface RemoveTenantOIDCAuthIntegrationResultObject {
   tenant?: Tenant;
   integration?: Omit<GQLOIDCAuthIntegration, "callbackURL">;
-  wasDeleted: boolean;
+  wasRemoved: boolean;
 }
 
 /**
- * deleteTenantOIDCAuthIntegration will delete the specific OpenID Connect Auth
+ * removeTenantOIDCAuthIntegration will delete the specific OpenID Connect Auth
  * Integration on the Tenant.
  *
  * @param mongo MongoDB Database handle
  * @param id the id of the Tenant
  * @param oidcID the id of the OpenID Connect Auth Integration we're deleting
  */
-export async function deleteTenantOIDCAuthIntegration(
+export async function removeTenantOIDCAuthIntegration(
   mongo: Db,
   id: string,
   oidcID: string
-): Promise<DeleteTenantOIDCAuthIntegrationResultObject> {
+): Promise<RemoveTenantOIDCAuthIntegrationResultObject> {
   const result = await collection(mongo).findOneAndUpdate(
     { id },
     {
@@ -384,7 +384,7 @@ export async function deleteTenantOIDCAuthIntegration(
     }
   );
   if (!result.value) {
-    return { wasDeleted: false };
+    return { wasRemoved: false };
   }
 
   // Find the integration that we wanted to delete.
@@ -394,7 +394,7 @@ export async function deleteTenantOIDCAuthIntegration(
   if (!integration) {
     // The integration was not in the original document, so we could not have
     // possibly deleted it!
-    return { wasDeleted: false };
+    return { wasRemoved: false };
   }
 
   // The integration was found, we should pull that integration out of the
@@ -406,6 +406,6 @@ export async function deleteTenantOIDCAuthIntegration(
   return {
     tenant: result.value,
     integration,
-    wasDeleted: true,
+    wasRemoved: true,
   };
 }

src/core/server/queue/index.ts

@@ -2,15 +2,12 @@ import Queue from "bull";
 import { Db } from "mongodb";
 
 import { Config } from "talk-common/config";
-import Task from "talk-server/services/queue/Task";
-import {
-  createMailerTask,
-  Mailer,
-} from "talk-server/services/queue/tasks/mailer";
+import Task from "talk-server/queue/Task";
+import { createMailerTask, Mailer } from "talk-server/queue/tasks/mailer";
 import {
   createScraperTask,
   ScraperData,
-} from "talk-server/services/queue/tasks/scraper";
+} from "talk-server/queue/tasks/scraper";
 import { createRedisClient } from "talk-server/services/redis";
 import TenantCache from "talk-server/services/tenant/cache";
 

src/core/server/queue/tasks/mailer/index.ts

@@ -6,8 +6,8 @@ import { createTransport } from "nodemailer";
 
 import { Config } from "talk-common/config";
 import logger from "talk-server/logger";
-import Task from "talk-server/services/queue/Task";
-import MailerContent from "talk-server/services/queue/tasks/mailer/content";
+import Task from "talk-server/queue/Task";
+import MailerContent from "talk-server/queue/tasks/mailer/content";
 import TenantCache from "talk-server/services/tenant/cache";
 import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter";
 

src/core/server/queue/tasks/scraper/index.ts

@@ -0,0 +1,55 @@
+import Queue, { Job } from "bull";
+import { Db } from "mongodb";
+
+import logger from "talk-server/logger";
+import Task from "talk-server/queue/Task";
+import { scrape } from "talk-server/services/stories/scraper";
+
+const JOB_NAME = "scraper";
+
+export interface ScrapeProcessorOptions {
+  mongo: Db;
+}
+
+export interface ScraperData {
+  storyID: string;
+  storyURL: string;
+  tenantID: string;
+}
+
+const createJobProcessor = ({ mongo }: ScrapeProcessorOptions) => async (
+  job: Job<ScraperData>
+) => {
+  // Pull out the job data.
+  const { storyID, storyURL, tenantID } = job.data;
+
+  const log = logger.child({
+    job_id: job.id,
+    job_name: JOB_NAME,
+    story_id: storyID,
+    story_url: storyURL,
+    tenant_id: tenantID,
+  });
+
+  log.debug("starting to scrape the story");
+
+  try {
+    await scrape(mongo, tenantID, storyID, storyURL);
+    log.debug("scraped the story");
+  } catch (err) {
+    log.error({ err }, "could not scrape the story");
+
+    throw err;
+  }
+};
+
+export function createScraperTask(
+  queue: Queue.QueueOptions,
+  options: ScrapeProcessorOptions
+) {
+  return new Task({
+    jobName: JOB_NAME,
+    jobProcessor: createJobProcessor(options),
+    queue,
+  });
+}

src/core/server/services/comments/actions.ts

@@ -1,15 +1,16 @@
 import { Db } from "mongodb";
 
+import { Omit } from "talk-common/types";
 import { GQLCOMMENT_FLAG_REPORTED_REASON } from "talk-server/graph/tenant/schema/__generated__/types";
 import {
   ACTION_ITEM_TYPE,
   ACTION_TYPE,
   CreateActionInput,
   createActions,
-  deleteAction,
-  DeleteActionInput,
   encodeActionCounts,
   invertEncodedActionCounts,
+  removeAction,
+  RemoveActionInput,
 } from "talk-server/models/action";
 import {
   retrieveComment,
@@ -20,11 +21,14 @@ import { updateStoryActionCounts } from "talk-server/models/story";
 import { Tenant } from "talk-server/models/tenant";
 import { User } from "talk-server/models/user";
 
+export type CreateAction = Omit<CreateActionInput, "root_item_id"> &
+  Required<Pick<CreateActionInput, "root_item_id">>;
+
 export async function addCommentActions(
   mongo: Db,
   tenant: Tenant,
   comment: Readonly<Comment>,
-  inputs: CreateActionInput[]
+  inputs: CreateAction[]
 ): Promise<Readonly<Comment>> {
   // Create each of the actions, returning each of the action results.
   const results = await createActions(mongo, tenant.id, inputs);
@@ -79,13 +83,19 @@ async function addCommentAction(
     throw new Error("comment not found");
   }
 
-  return addCommentActions(mongo, tenant, comment, [input]);
+  // Store the story ID on the action as a story_id.
+  input.root_item_id = comment.story_id;
+
+  // We have to perform a type assertion here because for some reason, the type
+  // coercion is not determining that because we filled in the `root_item_id`
+  // above, that at this point, it satisfies the CreateAction type.
+  return addCommentActions(mongo, tenant, comment, [input as CreateAction]);
 }
 
 export async function removeCommentAction(
   mongo: Db,
   tenant: Tenant,
-  input: DeleteActionInput
+  input: RemoveActionInput
 ): Promise<Readonly<Comment>> {
   // Get the Comment that we are leaving the Action on.
   const comment = await retrieveComment(mongo, tenant.id, input.item_id);
@@ -95,8 +105,8 @@ export async function removeCommentAction(
   }
 
   // Create each of the actions, returning each of the action results.
-  const { wasDeleted, action } = await deleteAction(mongo, tenant.id, input);
-  if (wasDeleted) {
+  const { wasRemoved, action } = await removeAction(mongo, tenant.id, input);
+  if (wasRemoved) {
     // Compute the action counts, and invert them (because we're deleting an
     // action).
     const actionCounts = invertEncodedActionCounts(encodeActionCounts(action!));
@@ -145,13 +155,13 @@ export async function createReaction(
   });
 }
 
-export type DeleteCommentReaction = Pick<DeleteActionInput, "item_id">;
+export type RemoveCommentReaction = Pick<RemoveActionInput, "item_id">;
 
-export async function deleteReaction(
+export async function removeReaction(
   mongo: Db,
   tenant: Tenant,
   author: User,
-  input: DeleteCommentReaction
+  input: RemoveCommentReaction
 ) {
   return removeCommentAction(mongo, tenant, {
     action_type: ACTION_TYPE.REACTION,
@@ -177,13 +187,13 @@ export async function createDontAgree(
   });
 }
 
-export type DeleteCommentDontAgree = Pick<DeleteActionInput, "item_id">;
+export type RemoveCommentDontAgree = Pick<RemoveActionInput, "item_id">;
 
-export async function deleteDontAgree(
+export async function removeDontAgree(
   mongo: Db,
   tenant: Tenant,
   author: User,
-  input: DeleteCommentDontAgree
+  input: RemoveCommentDontAgree
 ) {
   return removeCommentAction(mongo, tenant, {
     action_type: ACTION_TYPE.DONT_AGREE,
@@ -212,13 +222,13 @@ export async function createFlag(
   });
 }
 
-export type DeleteCommentFlag = Pick<DeleteActionInput, "item_id">;
+export type RemoveCommentFlag = Pick<RemoveActionInput, "item_id">;
 
-export async function deleteFlag(
+export async function removeFlag(
   mongo: Db,
   tenant: Tenant,
   author: User,
-  input: DeleteCommentFlag
+  input: RemoveCommentFlag
 ) {
   return removeCommentAction(mongo, tenant, {
     action_type: ACTION_TYPE.FLAG,

src/core/server/services/comments/index.ts

@@ -1,7 +1,7 @@
 import { Db } from "mongodb";
 
 import { Omit } from "talk-common/types";
-import { ACTION_ITEM_TYPE, CreateActionInput } from "talk-server/models/action";
+import { ACTION_ITEM_TYPE } from "talk-server/models/action";
 import {
   createComment,
   CreateCommentInput,
@@ -16,7 +16,10 @@ import {
 } from "talk-server/models/story";
 import { Tenant } from "talk-server/models/tenant";
 import { User } from "talk-server/models/user";
-import { addCommentActions } from "talk-server/services/comments/actions";
+import {
+  addCommentActions,
+  CreateAction,
+} from "talk-server/services/comments/actions";
 import { processForModeration } from "talk-server/services/comments/moderation";
 import { Request } from "talk-server/types/express";
 
@@ -83,10 +86,13 @@ export async function create(
     // at the time, and we didn't want the repetitive nature of adding the
     // item_type each time, so this mapping function adds them!
     const inputs = actions.map(
-      (action): CreateActionInput => ({
+      (action): CreateAction => ({
         ...action,
         item_id: comment.id,
         item_type: ACTION_ITEM_TYPE.COMMENTS,
+
+        // Store the Story ID on the action.
+        root_item_id: story.id,
       })
     );
 
@@ -171,12 +177,15 @@ export async function edit(
     // at the time, and we didn't want the repetitive nature of adding the
     // item_type each time, so this mapping function adds them!
     const inputs = actions.map(
-      (action): CreateActionInput => ({
+      (action): CreateAction => ({
         ...action,
         // Strict null check seems to have failed here... Null checking was done
         // above where we errored if the comment was falsely.
         item_id: comment!.id,
         item_type: ACTION_ITEM_TYPE.COMMENTS,
+
+        // Store the Story ID on the action.
+        root_item_id: story.id,
       })
     );
 

src/core/server/services/stories/index.ts

@@ -1,3 +1,4 @@
+import { zip } from "lodash";
 import { Db } from "mongodb";
 
 import {
@@ -8,12 +9,36 @@ import {
 } from "talk-server/app/url";
 import logger from "talk-server/logger";
 import {
+  countTotalActionCounts,
+  mergeActionCounts,
+  mergeManyRootActions,
+  removeRootActions,
+} from "talk-server/models/action";
+import {
+  mergeManyCommentStories,
+  removeStoryComments,
+} from "talk-server/models/comment";
+import {
+  calculateTotalCommentCount,
+  createStory,
+  CreateStoryInput,
   findOrCreateStory,
   FindOrCreateStoryInput,
+  mergeCommentStatusCount,
+  removeStories,
+  removeStory,
+  retrieveManyStories,
+  retrieveStory,
+  Story,
+  updateCommentStatusCount,
+  updateStory,
+  updateStoryActionCounts,
+  UpdateStoryInput,
 } from "talk-server/models/story";
 import { Tenant } from "talk-server/models/tenant";
-import Task from "talk-server/services/queue/Task";
-import { ScraperData } from "talk-server/services/queue/tasks/scraper";
+import Task from "talk-server/queue/Task";
+import { ScraperData } from "talk-server/queue/tasks/scraper";
+import { scrape } from "talk-server/services/stories/scraper";
 
 export type FindOrCreateStory = FindOrCreateStoryInput;
 
@@ -27,21 +52,24 @@ export async function findOrCreate(
   // to create the Asset.
   if (input.url && !isURLPermitted(tenant, input.url)) {
     logger.warn(
-      { storyURL: input.url, tenantDomains: tenant.domains },
-      "provided story url was not in the list of permitted tenant domains"
+      { story_url: input.url, tenant_domains: tenant.domains },
+      "provided story url was not in the list of permitted tenant domains, story not found"
     );
     return null;
   }
 
-  // TODO: check to see if the tenant has enabled lazy story creation.
+  // TODO: check to see if the tenant has enabled lazy story creation, if they haven't, switch to find only.
 
   const story = await findOrCreateStory(db, tenant.id, input);
   if (!story) {
     return null;
   }
 
-  if (!story.scrapedAt) {
-    // If the scraper has not scraped this story, we need to scrape it now!
+  // TODO: check to see if the tenant has scraping enabled.
+
+  if (!story.metadata && !story.scrapedAt) {
+    // If the scraper has not scraped this story, and we have no metadata, we
+    // need to scrape it now!
     await scraper.add({
       storyID: story.id,
       storyURL: story.url,
@@ -86,3 +114,235 @@ export function isURLPermitted(
     .map(domain => getOrigin(prefixSchemeIfRequired(originSecure, domain)))
     .some(origin => origin === targetOrigin);
 }
+
+export async function remove(
+  mongo: Db,
+  tenant: Tenant,
+  storyID: string,
+  includeComments: boolean = false
+) {
+  // Create a logger for this function.
+  const log = logger.child({
+    story_id: storyID,
+    include_comments: includeComments,
+  });
+
+  log.debug("starting to remove story");
+
+  // Get the story so we can see if there are associated comments.
+  const story = await retrieveStory(mongo, tenant.id, storyID);
+  if (!story) {
+    // No story was found!
+    log.warn("attempted to remove story that wasn't found");
+    return null;
+  }
+
+  if (includeComments) {
+    let removedCount: number | undefined;
+
+    // Remove the actions associated with the comments we just removed.
+    ({ deletedCount: removedCount } = await removeRootActions(
+      mongo,
+      tenant.id,
+      story.id
+    ));
+
+    log.debug(
+      { removed_actions: removedCount },
+      "removed actions while deleting story"
+    );
+
+    // Remove the comments for the story.
+    ({ deletedCount: removedCount } = await removeStoryComments(
+      mongo,
+      tenant.id,
+      story.id
+    ));
+
+    log.debug(
+      { removed_comments: removedCount },
+      "removed comments while deleting story"
+    );
+  } else if (calculateTotalCommentCount(story.comment_counts) > 0) {
+    log.warn(
+      "attempted to remove story that has linked comments without consent for deleting comments"
+    );
+
+    // TODO: (wyattjoh) improve error
+    throw new Error("asset has comments, cannot remove");
+  }
+
+  const removedStory = await removeStory(mongo, tenant.id, story.id);
+  if (!removedStory) {
+    // Story was already removed.
+    // TODO: evaluate use of transaction here.
+    return null;
+  }
+
+  log.debug("removed story");
+
+  return removedStory;
+}
+
+export type CreateStory = CreateStoryInput;
+
+export async function create(
+  mongo: Db,
+  tenant: Tenant,
+  storyID: string,
+  storyURL: string,
+  input: CreateStory
+) {
+  // Ensure that the given URL is allowed.
+  if (!isURLPermitted(tenant, storyURL)) {
+    logger.warn(
+      { story_url: storyURL, tenant_domains: tenant.domains },
+      "provided story url was not in the list of permitted tenant domains, story not created"
+    );
+    return null;
+  }
+
+  // Create the story in the database.
+  let newStory = await createStory(mongo, tenant.id, storyID, storyURL, input);
+  if (!input.metadata && !newStory.scrapedAt) {
+    // If the scraper has not scraped this story and story metadata was not
+    // provided, we need to scrape it now!
+    newStory = await scrape(mongo, tenant.id, newStory.id);
+  }
+
+  return newStory;
+}
+
+export type UpdateStory = UpdateStoryInput;
+
+export async function update(
+  mongo: Db,
+  tenant: Tenant,
+  storyID: string,
+  input: UpdateStory
+) {
+  // Ensure that the given URL is allowed.
+  if (input.url && !isURLPermitted(tenant, input.url)) {
+    logger.warn(
+      { story_url: input.url, tenant_domains: tenant.domains },
+      "provided story url was not in the list of permitted tenant domains, story not updated"
+    );
+    return null;
+  }
+
+  return updateStory(mongo, tenant.id, storyID, input);
+}
+
+export async function merge(
+  mongo: Db,
+  tenant: Tenant,
+  destinationID: string,
+  sourceIDs: string[]
+) {
+  // Create a logger for this operation.
+  const log = logger.child({
+    destination_id: destinationID,
+    source_ids: sourceIDs,
+  });
+
+  if (sourceIDs.length === 0) {
+    log.warn("cannot merge from 0 stories");
+    return null;
+  }
+
+  // Get the stories referenced.
+  const storyIDs = [destinationID, ...sourceIDs];
+  const stories = await retrieveManyStories(mongo, tenant.id, storyIDs);
+
+  // Ensure that these are all defined.
+  if (
+    zip(storyIDs, stories).some(([storyID, story]) => {
+      if (!story) {
+        log.warn(
+          { story_id: storyID },
+          "story that was going to be merged was not found"
+        );
+        return true;
+      }
+
+      return false;
+    })
+  ) {
+    return null;
+  }
+
+  let updatedCount: number | undefined;
+
+  // Move all the comment's from the source stories over to the destination
+  // story.
+  ({ modifiedCount: updatedCount } = await mergeManyCommentStories(
+    mongo,
+    tenant.id,
+    destinationID,
+    sourceIDs
+  ));
+
+  log.debug(
+    { updated_comments: updatedCount },
+    "updated comments while merging stories"
+  );
+
+  // Update all the action's that referenced the old story to reference the new
+  // story.
+  ({ modifiedCount: updatedCount } = await mergeManyRootActions(
+    mongo,
+    tenant.id,
+    destinationID,
+    sourceIDs
+  ));
+
+  log.debug(
+    { updated_actions: updatedCount },
+    "updated actions while merging stories"
+  );
+
+  // Merge the comment and action counts for all the source stories.
+  const [, ...sourceStories] = stories;
+
+  let destinationStory = await updateCommentStatusCount(
+    mongo,
+    tenant.id,
+    destinationID,
+    mergeCommentStatusCount(
+      // We perform the type assertion here because above, we already verified
+      // that none of the stories are null.
+      (sourceStories as Story[]).map(({ comment_counts }) => comment_counts)
+    )
+  );
+
+  const mergedActionCounts = mergeActionCounts(
+    // We perform the type assertion here because above, we already verified
+    // that none of the stories are null.
+    (sourceStories as Story[]).map(({ action_counts }) => action_counts)
+  );
+  if (countTotalActionCounts(mergedActionCounts) > 0) {
+    destinationStory = await updateStoryActionCounts(
+      mongo,
+      tenant.id,
+      destinationID,
+      mergedActionCounts
+    );
+  }
+
+  if (!destinationStory) {
+    log.warn("destination story cannot be updated with new comment counts");
+    return null;
+  }
+
+  log.debug(
+    { comment_counts: destinationStory.comment_counts },
+    "updated destination story with new comment counts"
+  );
+
+  const { deletedCount } = await removeStories(mongo, tenant.id, sourceIDs);
+
+  log.debug({ deleted_stories: deletedCount }, "deleted source stories");
+
+  // Return the story that had the other stories merged into.
+  return destinationStory;
+}

src/core/server/services/stories/scraper/index.ts

@@ -1,4 +1,3 @@
-import Queue, { Job } from "bull";
 import Logger from "bunyan";
 import cheerio from "cheerio";
 import authorScraper from "metascraper-author";
@@ -10,87 +9,11 @@ import { Db } from "mongodb";
 
 import { GQLStoryMetadata } from "talk-server/graph/tenant/schema/__generated__/types";
 import logger from "talk-server/logger";
-import { updateStory } from "talk-server/models/story";
-import Task from "talk-server/services/queue/Task";
+import { retrieveStory, updateStory } from "talk-server/models/story";
+
 import { modifiedScraper } from "./rules/modified";
 import { sectionScraper } from "./rules/section";
 
-const JOB_NAME = "scraper";
-
-export interface ScrapeProcessorOptions {
-  mongo: Db;
-}
-
-export interface ScraperData {
-  storyID: string;
-  storyURL: string;
-  tenantID: string;
-}
-
-const createJobProcessor = (options: ScrapeProcessorOptions) => async (
-  job: Job<ScraperData>
-) => {
-  // Pull out the job data.
-  const { storyID: id, storyURL: url, tenantID } = job.data;
-
-  logger.debug(
-    {
-      job_id: job.id,
-      job_name: JOB_NAME,
-      story_id: id,
-      story_url: url,
-      tenant_id: tenantID,
-    },
-    "starting to scrap the story"
-  );
-
-  // Get the metadata from the scraped html.
-  const metadata = await scraper.scrape(url);
-  if (!metadata) {
-    logger.error(
-      {
-        job_id: job.id,
-        job_name: JOB_NAME,
-        story_id: id,
-        story_url: url,
-        tenant_id: tenantID,
-      },
-      "story at specified url not found, can not scrape"
-    );
-    return;
-  }
-
-  // Update the Story with the scraped details.
-  const story = await updateStory(options.mongo, tenantID, id, {
-    metadata,
-    scrapedAt: new Date(),
-  });
-  if (!story) {
-    logger.error(
-      {
-        job_id: job.id,
-        job_name: JOB_NAME,
-        story_id: id,
-        story_url: url,
-        tenant_id: tenantID,
-      },
-      "story at specified id not found, can not update with metadata"
-    );
-    return;
-  }
-
-  logger.debug(
-    {
-      job_id: job.id,
-      job_name: JOB_NAME,
-      story_id: story.id,
-      story_url: url,
-      tenant_id: tenantID,
-    },
-    "scraped the story"
-  );
-};
-
 export type Rule = Record<
   string,
   Array<
@@ -188,13 +111,37 @@ function createScraper() {
 
 export const scraper = createScraper();
 
-export function createScraperTask(
-  queue: Queue.QueueOptions,
-  options: ScrapeProcessorOptions
+export async function scrape(
+  mongo: Db,
+  tenantID: string,
+  storyID: string,
+  storyURL?: string
 ) {
-  return new Task({
-    jobName: JOB_NAME,
-    jobProcessor: createJobProcessor(options),
-    queue,
+  // If the URL wasn't provided, grab it from the database.
+  if (!storyURL) {
+    const retrievedStory = await retrieveStory(mongo, tenantID, storyID);
+    if (!retrievedStory) {
+      throw new Error("story at specified id not found");
+    }
+
+    // Update the story URL.
+    storyURL = retrievedStory.url;
+  }
+
+  // Get the metadata from the scraped html.
+  const metadata = await scraper.scrape(storyURL);
+  if (!metadata) {
+    throw new Error("story at specified url not found");
+  }
+
+  // Update the Story with the scraped details.
+  const story = await updateStory(mongo, tenantID, storyID, {
+    metadata,
+    scrapedAt: new Date(),
   });
+  if (!story) {
+    throw new Error("story at specified id not found");
+  }
+
+  return story;
 }

src/core/server/services/tenant/index.ts

@@ -11,8 +11,8 @@ import {
   createTenant,
   CreateTenantInput,
   createTenantOIDCAuthIntegration,
-  deleteTenantOIDCAuthIntegration,
   regenerateTenantSSOKey,
+  removeTenantOIDCAuthIntegration,
   Tenant,
   updateTenant,
   updateTenantOIDCAuthIntegration,
@@ -176,7 +176,7 @@ export async function updateOIDCAuthIntegration(
   return result;
 }
 
-export async function deleteOIDCAuthIntegration(
+export async function removeOIDCAuthIntegration(
   mongo: Db,
   redis: Redis,
   cache: TenantCache,
@@ -184,12 +184,12 @@ export async function deleteOIDCAuthIntegration(
   oidcID: string
 ) {
   // Delete the integration. By default, the integration is disabled.
-  const result = await deleteTenantOIDCAuthIntegration(
+  const result = await removeTenantOIDCAuthIntegration(
     mongo,
     tenant.id,
     oidcID
   );
-  if (!result.wasDeleted || !result.tenant) {
+  if (!result.wasRemoved || !result.tenant) {
     return null;
   }