From ca12b73b55e703d3951e79a35e78cd6959391846 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Fri, 2 Nov 2018 23:18:12 +0000 Subject: [PATCH] [next] Story Mutations (#2055) * feat: added mutations for removing + scraping stories - added mutation for removing stories - added mutation for scraping stories - renamed all delete* to remove* to avoid reserved keyword collision * fix: linting * feat: implemented createStory * feat: added support for createStory * feat: improved Story deletion - Deletes comments on the deleted Story - Deletes actions of comments that were deleted via new `root_item_id` parameter for Comment Action's - Added logging around critical components of the remove process * feat: implemented mergeStory - Added new merge function for root actions - Added new merge function for story comments - Added new merge function for comment status counts - Added new remove stories function * fix: fixed story creation method * fix: added action count merging * fix: removed outdated file * fix: removed queue that got duplicated via rebase * fix: prevent error when action counts are empty --- ...on.ts => RemoveCommentReactionMutation.ts} | 18 +- src/core/client/stream/mutations/index.ts | 6 +- .../containers/ReactionButtonContainer.tsx | 12 +- src/core/server/app/index.ts | 2 +- .../server/app/middleware/context/tenant.ts | 2 +- src/core/server/graph/tenant/context.ts | 2 +- .../server/graph/tenant/loaders/stories.ts | 2 +- .../server/graph/tenant/mutators/comment.ts | 24 +- .../server/graph/tenant/mutators/index.ts | 2 + .../server/graph/tenant/mutators/settings.ts | 8 +- .../server/graph/tenant/mutators/story.ts | 32 ++ .../server/graph/tenant/resolvers/mutation.ts | 36 +- .../server/graph/tenant/schema/schema.graphql | 339 ++++++++++++++++-- src/core/server/index.ts | 2 +- src/core/server/models/action.ts | 127 ++++++- src/core/server/models/comment.ts | 40 +++ src/core/server/models/story.ts | 157 +++++++- src/core/server/models/tenant.ts | 16 +- src/core/server/{services => }/queue/Task.ts | 0 src/core/server/{services => }/queue/index.ts | 9 +- .../server/{services => }/queue/tasks/Task.ts | 0 .../queue/tasks/mailer/content.ts | 0 .../queue/tasks/mailer/index.ts | 4 +- .../mailer/templates/forgot-password.html | 0 src/core/server/queue/tasks/scraper/index.ts | 55 +++ src/core/server/services/comments/actions.ts | 42 ++- src/core/server/services/comments/index.ts | 17 +- src/core/server/services/stories/index.ts | 274 +++++++++++++- .../{queue/tasks => stories}/scraper/index.ts | 119 ++---- .../scraper/rules/modified.ts | 0 .../scraper/rules/section.ts | 0 src/core/server/services/tenant/index.ts | 8 +- 32 files changed, 1123 insertions(+), 232 deletions(-) rename src/core/client/stream/mutations/{DeleteCommentReactionMutation.ts => RemoveCommentReactionMutation.ts} (70%) create mode 100644 src/core/server/graph/tenant/mutators/story.ts rename src/core/server/{services => }/queue/Task.ts (100%) rename src/core/server/{services => }/queue/index.ts (90%) rename src/core/server/{services => }/queue/tasks/Task.ts (100%) rename src/core/server/{services => }/queue/tasks/mailer/content.ts (100%) rename src/core/server/{services => }/queue/tasks/mailer/index.ts (97%) rename src/core/server/{services => }/queue/tasks/mailer/templates/forgot-password.html (100%) create mode 100644 src/core/server/queue/tasks/scraper/index.ts rename src/core/server/services/{queue/tasks => stories}/scraper/index.ts (65%) rename src/core/server/services/{queue/tasks => stories}/scraper/rules/modified.ts (100%) rename src/core/server/services/{queue/tasks => stories}/scraper/rules/section.ts (100%) diff --git a/src/core/client/stream/mutations/DeleteCommentReactionMutation.ts b/src/core/client/stream/mutations/RemoveCommentReactionMutation.ts similarity index 70% rename from src/core/client/stream/mutations/DeleteCommentReactionMutation.ts rename to src/core/client/stream/mutations/RemoveCommentReactionMutation.ts index 027723544..1ed7a0de4 100644 --- a/src/core/client/stream/mutations/DeleteCommentReactionMutation.ts +++ b/src/core/client/stream/mutations/RemoveCommentReactionMutation.ts @@ -6,12 +6,12 @@ import { createMutationContainer, } from "talk-framework/lib/relay"; -import { DeleteCommentReactionMutation as MutationTypes } from "talk-stream/__generated__/DeleteCommentReactionMutation.graphql"; -import { CreateCommentReactionInput } from "./CreateCommentReactionMutation"; +import { RemoveCommentReactionMutation as MutationTypes } from "talk-stream/__generated__/RemoveCommentReactionMutation.graphql"; +import { CreateCommentReactionInput } from "talk-stream/mutations/CreateCommentReactionMutation"; const mutation = graphql` - mutation DeleteCommentReactionMutation($input: CreateCommentReactionInput!) { - deleteCommentReaction(input: $input) { + mutation RemoveCommentReactionMutation($input: CreateCommentReactionInput!) { + removeCommentReaction(input: $input) { comment { ...ReactionButtonContainer_comment } @@ -36,7 +36,7 @@ function commit(environment: Environment, input: CreateCommentReactionInput) { }, }, optimisticResponse: { - deleteCommentReaction: { + removeCommentReaction: { comment: { id: input.commentID, myActionPresence: { @@ -54,11 +54,11 @@ function commit(environment: Environment, input: CreateCommentReactionInput) { }); } -export const withDeleteCommentReactionMutation = createMutationContainer( - "deleteCommentReaction", +export const withRemoveCommentReactionMutation = createMutationContainer( + "removeCommentReaction", commit ); -export type DeleteCommentReactionMutation = ( +export type RemoveCommentReactionMutation = ( input: CreateCommentReactionInput -) => Promise; +) => Promise; diff --git a/src/core/client/stream/mutations/index.ts b/src/core/client/stream/mutations/index.ts index 648ae90cc..5fa3e2e98 100644 --- a/src/core/client/stream/mutations/index.ts +++ b/src/core/client/stream/mutations/index.ts @@ -32,6 +32,6 @@ export { CreateCommentReactionInput, } from "./CreateCommentReactionMutation"; export { - withDeleteCommentReactionMutation, - DeleteCommentReactionMutation, -} from "./DeleteCommentReactionMutation"; + withRemoveCommentReactionMutation, + RemoveCommentReactionMutation, +} from "./RemoveCommentReactionMutation"; diff --git a/src/core/client/stream/tabs/comments/containers/ReactionButtonContainer.tsx b/src/core/client/stream/tabs/comments/containers/ReactionButtonContainer.tsx index 26d44a6c1..43b4f0fce 100644 --- a/src/core/client/stream/tabs/comments/containers/ReactionButtonContainer.tsx +++ b/src/core/client/stream/tabs/comments/containers/ReactionButtonContainer.tsx @@ -7,9 +7,9 @@ import { ReactionButtonContainer_settings as SettingsData } from "talk-stream/__ import { CreateCommentReactionMutation, - DeleteCommentReactionMutation, + RemoveCommentReactionMutation, withCreateCommentReactionMutation, - withDeleteCommentReactionMutation, + withRemoveCommentReactionMutation, } from "talk-stream/mutations"; import ReactionButton from "talk-stream/tabs/comments/components/ReactionButton"; @@ -20,7 +20,7 @@ import { interface ReactionButtonContainerProps { createCommentReaction: CreateCommentReactionMutation; - deleteCommentReaction: DeleteCommentReactionMutation; + removeCommentReaction: RemoveCommentReactionMutation; comment: CommentData; settings: SettingsData; me: MeData | null; @@ -41,13 +41,13 @@ class ReactionButtonContainer extends React.Component< commentID: this.props.comment.id, }; - const { createCommentReaction, deleteCommentReaction } = this.props; + const { createCommentReaction, removeCommentReaction } = this.props; const reacted = this.props.comment.myActionPresence && this.props.comment.myActionPresence.reaction; return reacted - ? deleteCommentReaction(input) + ? removeCommentReaction(input) : createCommentReaction(input); }; public render() { @@ -79,7 +79,7 @@ class ReactionButtonContainer extends React.Component< } export default withShowAuthPopupMutation( - withDeleteCommentReactionMutation( + withRemoveCommentReactionMutation( withCreateCommentReactionMutation( withFragmentContainer({ me: graphql` diff --git a/src/core/server/app/index.ts b/src/core/server/app/index.ts index c4ae82ab3..d9d6160a8 100644 --- a/src/core/server/app/index.ts +++ b/src/core/server/app/index.ts @@ -14,8 +14,8 @@ import { notFoundMiddleware } from "talk-server/app/middleware/notFound"; import { createPassport } from "talk-server/app/middleware/passport"; import { handleSubscriptions } from "talk-server/graph/common/subscriptions/middleware"; import { Schemas } from "talk-server/graph/schemas"; +import { TaskQueue } from "talk-server/queue"; import { JWTSigningConfig } from "talk-server/services/jwt"; -import { TaskQueue } from "talk-server/services/queue"; import TenantCache from "talk-server/services/tenant/cache"; import { accessLogger, errorLogger } from "./middleware/logging"; diff --git a/src/core/server/app/middleware/context/tenant.ts b/src/core/server/app/middleware/context/tenant.ts index 549e70266..f887ac71d 100644 --- a/src/core/server/app/middleware/context/tenant.ts +++ b/src/core/server/app/middleware/context/tenant.ts @@ -4,7 +4,7 @@ import { Db } from "mongodb"; import { Config } from "talk-common/config"; import TenantContext from "talk-server/graph/tenant/context"; -import { TaskQueue } from "talk-server/services/queue"; +import { TaskQueue } from "talk-server/queue"; import { Request } from "talk-server/types/express"; export interface TenantContextMiddlewareOptions { diff --git a/src/core/server/graph/tenant/context.ts b/src/core/server/graph/tenant/context.ts index d8785338b..03f30d72a 100644 --- a/src/core/server/graph/tenant/context.ts +++ b/src/core/server/graph/tenant/context.ts @@ -4,7 +4,7 @@ import { Db } from "mongodb"; import CommonContext from "talk-server/graph/common/context"; import { Tenant } from "talk-server/models/tenant"; import { User } from "talk-server/models/user"; -import { TaskQueue } from "talk-server/services/queue"; +import { TaskQueue } from "talk-server/queue"; import TenantCache from "talk-server/services/tenant/cache"; import { Request } from "talk-server/types/express"; diff --git a/src/core/server/graph/tenant/loaders/stories.ts b/src/core/server/graph/tenant/loaders/stories.ts index 0a9efb9e7..47c946618 100644 --- a/src/core/server/graph/tenant/loaders/stories.ts +++ b/src/core/server/graph/tenant/loaders/stories.ts @@ -7,8 +7,8 @@ import { retrieveManyStories, Story, } from "talk-server/models/story"; -import { scraper } from "talk-server/services/queue/tasks/scraper"; import { findOrCreate } from "talk-server/services/stories"; +import { scraper } from "talk-server/services/stories/scraper"; export default (ctx: TenantContext) => ({ findOrCreate: (input: FindOrCreateStoryInput) => diff --git a/src/core/server/graph/tenant/mutators/comment.ts b/src/core/server/graph/tenant/mutators/comment.ts index 6d092fe4c..e61377839 100644 --- a/src/core/server/graph/tenant/mutators/comment.ts +++ b/src/core/server/graph/tenant/mutators/comment.ts @@ -4,19 +4,19 @@ import { GQLCreateCommentFlagInput, GQLCreateCommentInput, GQLCreateCommentReactionInput, - GQLDeleteCommentDontAgreeInput, - GQLDeleteCommentFlagInput, - GQLDeleteCommentReactionInput, GQLEditCommentInput, + GQLRemoveCommentDontAgreeInput, + GQLRemoveCommentFlagInput, + GQLRemoveCommentReactionInput, } from "talk-server/graph/tenant/schema/__generated__/types"; import { create, edit } from "talk-server/services/comments"; import { createDontAgree, createFlag, createReaction, - deleteDontAgree, - deleteFlag, - deleteReaction, + removeDontAgree, + removeFlag, + removeReaction, } from "talk-server/services/comments/actions"; export default (ctx: TenantContext) => ({ @@ -48,16 +48,16 @@ export default (ctx: TenantContext) => ({ createReaction(ctx.mongo, ctx.tenant, ctx.user!, { item_id: input.commentID, }), - deleteReaction: (input: GQLDeleteCommentReactionInput) => - deleteReaction(ctx.mongo, ctx.tenant, ctx.user!, { + removeReaction: (input: GQLRemoveCommentReactionInput) => + removeReaction(ctx.mongo, ctx.tenant, ctx.user!, { item_id: input.commentID, }), createDontAgree: (input: GQLCreateCommentDontAgreeInput) => createDontAgree(ctx.mongo, ctx.tenant, ctx.user!, { item_id: input.commentID, }), - deleteDontAgree: (input: GQLDeleteCommentDontAgreeInput) => - deleteDontAgree(ctx.mongo, ctx.tenant, ctx.user!, { + removeDontAgree: (input: GQLRemoveCommentDontAgreeInput) => + removeDontAgree(ctx.mongo, ctx.tenant, ctx.user!, { item_id: input.commentID, }), createFlag: (input: GQLCreateCommentFlagInput) => @@ -65,8 +65,8 @@ export default (ctx: TenantContext) => ({ item_id: input.commentID, reason: input.reason, }), - deleteFlag: (input: GQLDeleteCommentFlagInput) => - deleteFlag(ctx.mongo, ctx.tenant, ctx.user!, { + removeFlag: (input: GQLRemoveCommentFlagInput) => + removeFlag(ctx.mongo, ctx.tenant, ctx.user!, { item_id: input.commentID, }), }); diff --git a/src/core/server/graph/tenant/mutators/index.ts b/src/core/server/graph/tenant/mutators/index.ts index 8ec9d1d02..fb4c940fd 100644 --- a/src/core/server/graph/tenant/mutators/index.ts +++ b/src/core/server/graph/tenant/mutators/index.ts @@ -2,8 +2,10 @@ import TenantContext from "talk-server/graph/tenant/context"; import Comment from "./comment"; import Settings from "./settings"; +import Story from "./story"; export default (ctx: TenantContext) => ({ Comment: Comment(ctx), Settings: Settings(ctx), + Story: Story(ctx), }); diff --git a/src/core/server/graph/tenant/mutators/settings.ts b/src/core/server/graph/tenant/mutators/settings.ts index 16c3ac6c0..b7ffc1846 100644 --- a/src/core/server/graph/tenant/mutators/settings.ts +++ b/src/core/server/graph/tenant/mutators/settings.ts @@ -3,15 +3,15 @@ import { isNull, omitBy } from "lodash"; import TenantContext from "talk-server/graph/tenant/context"; import { GQLCreateOIDCAuthIntegrationInput, - GQLDeleteOIDCAuthIntegrationInput, + GQLRemoveOIDCAuthIntegrationInput, GQLSettingsInput, GQLUpdateOIDCAuthIntegrationInput, } from "talk-server/graph/tenant/schema/__generated__/types"; import { Tenant } from "talk-server/models/tenant"; import { createOIDCAuthIntegration, - deleteOIDCAuthIntegration, regenerateSSOKey, + removeOIDCAuthIntegration, update, updateOIDCAuthIntegration, } from "talk-server/services/tenant"; @@ -38,6 +38,6 @@ export default ({ mongo, redis, tenantCache, tenant }: TenantContext) => ({ input.id, input.configuration ), - deleteOIDCAuthIntegration: (input: GQLDeleteOIDCAuthIntegrationInput) => - deleteOIDCAuthIntegration(mongo, redis, tenantCache, tenant, input.id), + removeOIDCAuthIntegration: (input: GQLRemoveOIDCAuthIntegrationInput) => + removeOIDCAuthIntegration(mongo, redis, tenantCache, tenant, input.id), }); diff --git a/src/core/server/graph/tenant/mutators/story.ts b/src/core/server/graph/tenant/mutators/story.ts new file mode 100644 index 000000000..b8909f832 --- /dev/null +++ b/src/core/server/graph/tenant/mutators/story.ts @@ -0,0 +1,32 @@ +import { isNull, omitBy } from "lodash"; + +import TenantContext from "talk-server/graph/tenant/context"; +import { + GQLCreateStoryInput, + GQLMergeStoriesInput, + GQLRemoveStoryInput, + GQLScrapeStoryInput, + GQLUpdateStoryInput, +} from "talk-server/graph/tenant/schema/__generated__/types"; +import { Story } from "talk-server/models/story"; +import { create, merge, remove, update } from "talk-server/services/stories"; +import { scrape } from "talk-server/services/stories/scraper"; + +export default (ctx: TenantContext) => ({ + create: async (input: GQLCreateStoryInput): Promise | null> => + create( + ctx.mongo, + ctx.tenant, + input.story.id, + input.story.url, + omitBy(input.story, isNull) + ), + update: async (input: GQLUpdateStoryInput): Promise | null> => + update(ctx.mongo, ctx.tenant, input.id, omitBy(input.story, isNull)), + merge: async (input: GQLMergeStoriesInput): Promise | null> => + merge(ctx.mongo, ctx.tenant, input.destinationID, input.sourceIDs), + remove: async (input: GQLRemoveStoryInput): Promise | null> => + remove(ctx.mongo, ctx.tenant, input.id, input.includeComments), + scrape: async (input: GQLScrapeStoryInput): Promise | null> => + scrape(ctx.mongo, ctx.tenant.id, input.id), +}); diff --git a/src/core/server/graph/tenant/resolvers/mutation.ts b/src/core/server/graph/tenant/resolvers/mutation.ts index 32425a67d..320da90dc 100644 --- a/src/core/server/graph/tenant/resolvers/mutation.ts +++ b/src/core/server/graph/tenant/resolvers/mutation.ts @@ -27,24 +27,24 @@ const Mutation: GQLMutationTypeResolver = { comment: await ctx.mutators.Comment.createReaction(input), clientMutationId: input.clientMutationId, }), - deleteCommentReaction: async (source, { input }, ctx) => ({ - comment: await ctx.mutators.Comment.deleteReaction(input), + removeCommentReaction: async (source, { input }, ctx) => ({ + comment: await ctx.mutators.Comment.removeReaction(input), clientMutationId: input.clientMutationId, }), createCommentDontAgree: async (source, { input }, ctx) => ({ comment: await ctx.mutators.Comment.createDontAgree(input), clientMutationId: input.clientMutationId, }), - deleteCommentDontAgree: async (source, { input }, ctx) => ({ - comment: await ctx.mutators.Comment.deleteDontAgree(input), + removeCommentDontAgree: async (source, { input }, ctx) => ({ + comment: await ctx.mutators.Comment.removeDontAgree(input), clientMutationId: input.clientMutationId, }), createCommentFlag: async (source, { input }, ctx) => ({ comment: await ctx.mutators.Comment.createFlag(input), clientMutationId: input.clientMutationId, }), - deleteCommentFlag: async (source, { input }, ctx) => ({ - comment: await ctx.mutators.Comment.deleteFlag(input), + removeCommentFlag: async (source, { input }, ctx) => ({ + comment: await ctx.mutators.Comment.removeFlag(input), clientMutationId: input.clientMutationId, }), regenerateSSOKey: async (source, { input }, ctx) => ({ @@ -75,8 +75,8 @@ const Mutation: GQLMutationTypeResolver = { clientMutationId: input.clientMutationId, }; }, - deleteOIDCAuthIntegration: async (source, { input }, ctx) => { - const result = await ctx.mutators.Settings.deleteOIDCAuthIntegration(input); + removeOIDCAuthIntegration: async (source, { input }, ctx) => { + const result = await ctx.mutators.Settings.removeOIDCAuthIntegration(input); if (!result) { return { clientMutationId: input.clientMutationId }; } @@ -87,6 +87,26 @@ const Mutation: GQLMutationTypeResolver = { clientMutationId: input.clientMutationId, }; }, + createStory: async (source, { input }, ctx) => ({ + story: await ctx.mutators.Story.create(input), + clientMutationId: input.clientMutationId, + }), + updateStory: async (source, { input }, ctx) => ({ + story: await ctx.mutators.Story.update(input), + clientMutationId: input.clientMutationId, + }), + mergeStories: async (source, { input }, ctx) => ({ + story: await ctx.mutators.Story.merge(input), + clientMutationId: input.clientMutationId, + }), + removeStory: async (source, { input }, ctx) => ({ + story: await ctx.mutators.Story.remove(input), + clientMutationId: input.clientMutationId, + }), + scrapeStory: async (source, { input }, ctx) => ({ + story: await ctx.mutators.Story.scrape(input), + clientMutationId: input.clientMutationId, + }), }; export default Mutation; diff --git a/src/core/server/graph/tenant/schema/schema.graphql b/src/core/server/graph/tenant/schema/schema.graphql index 241a9d4f8..c731e8c46 100644 --- a/src/core/server/graph/tenant/schema/schema.graphql +++ b/src/core/server/graph/tenant/schema/schema.graphql @@ -1052,7 +1052,7 @@ type Comment { """ replyCount is the number of replies. Only direct replies to this Comment - are counted. Deleted comments are included in this count. + are counted. Removed comments are included in this count. """ replyCount: Int! @@ -1918,12 +1918,12 @@ type CreateCommentReactionPayload { } ################## -## deleteCommentReaction +## removeCommentReaction ################## -input DeleteCommentReactionInput { +input RemoveCommentReactionInput { """ - commentID is the Comment's ID that we want to delete a Reaction on. + commentID is the Comment's ID that we want to remove a Reaction on. """ commentID: ID! @@ -1933,9 +1933,9 @@ input DeleteCommentReactionInput { clientMutationId: String! } -type DeleteCommentReactionPayload { +type RemoveCommentReactionPayload { """ - comment is the Comment that the Reaction was deleted on. + comment is the Comment that the Reaction was removed on. """ comment: Comment @@ -1974,12 +1974,12 @@ type CreateCommentDontAgreePayload { } ################## -## deleteCommentDontAgree +## removeCommentDontAgree ################## -input DeleteCommentDontAgreeInput { +input RemoveCommentDontAgreeInput { """ - commentID is the Comment's ID that we want to delete a DontAgree on. + commentID is the Comment's ID that we want to remove a DontAgree on. """ commentID: ID! @@ -1989,9 +1989,9 @@ input DeleteCommentDontAgreeInput { clientMutationId: String! } -type DeleteCommentDontAgreePayload { +type RemoveCommentDontAgreePayload { """ - comment is the Comment that the DontAgree was deleted on. + comment is the Comment that the DontAgree was removed on. """ comment: Comment @@ -2035,12 +2035,12 @@ type CreateCommentFlagPayload { } ################## -## deleteCommentFlag +## removeCommentFlag ################## -input DeleteCommentFlagInput { +input RemoveCommentFlagInput { """ - commentID is the Comment's ID that we want to delete a Flag on. + commentID is the Comment's ID that we want to remove a Flag on. """ commentID: ID! @@ -2050,9 +2050,9 @@ input DeleteCommentFlagInput { clientMutationId: String! } -type DeleteCommentFlagPayload { +type RemoveCommentFlagPayload { """ - comment is the Comment that the Flag was deleted on. + comment is the Comment that the Flag was removed on. """ comment: Comment @@ -2278,10 +2278,10 @@ type UpdateOIDCAuthIntegrationPayload { } ################## -# deleteOIDCAuthIntegration +# removeOIDCAuthIntegration ################## -input DeleteOIDCAuthIntegrationInput { +input RemoveOIDCAuthIntegrationInput { """ id is the ID of the specific OpenID Connect integration that we are deleting. """ @@ -2293,14 +2293,14 @@ input DeleteOIDCAuthIntegrationInput { clientMutationId: String! } -type DeleteOIDCAuthIntegrationPayload { +type RemoveOIDCAuthIntegrationPayload { """ - integration is the OIDCAuthIntegration we just deleted. + integration is the OIDCAuthIntegration we just removed. """ integration: OIDCAuthIntegration """ - settings is the Settings that the OIDCAuthIntegration was deleted on with the + settings is the Settings that the OIDCAuthIntegration was removed on with the OIDCAuthIntegration removed from it. """ settings: Settings @@ -2311,6 +2311,252 @@ type DeleteOIDCAuthIntegrationPayload { clientMutationId: String! } +################## +## createStory +################## + +""" +StoryMetadataInput is the metadata for a given Story as provided via this API. +""" +input StoryMetadataInput { + """ + title stores the title from the Story page. + """ + title: String + + """ + author stores the author from the Story page. + """ + author: String + + """ + description stores the description from the Story page. + """ + description: String + + """ + image stores the image from the Story page. + """ + image: String + + """ + publishedAt stores the publication date from the Story page. + """ + publishedAt: Time + + """ + modifiedAt stores the modified date from the Story page. + """ + modifiedAt: Time + + """ + section stores the section from the Story page. + """ + section: String +} + +""" +CreateStory is the input required to create a Story. +""" +input CreateStory { + """ + id is the identifier of the Story. + """ + id: ID! + + """ + url is the url that the Story is located on. + """ + url: String! + + """ + metadata is the set of information relating to this Story that would normally + be scraped, but can be provided here. + """ + metadata: StoryMetadataInput +} + +input CreateStoryInput { + """ + story is the Story input needed to create a Story. + """ + story: CreateStory! + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +type CreateStoryPayload { + """ + story is the Story that was possibly created. + """ + story: Story + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +################## +## updateStory +################## + +""" +UpdateStory is the input required to update a Story. +""" +input UpdateStory { + """ + url is the url that the Story is located on. + """ + url: String + + """ + metadata is the set of information relating to this Story that would normally + be scraped, but can be provided here. + """ + metadata: StoryMetadataInput + + """ + isClosed is true when the Story should be closed for commenting. + """ + isClosed: Boolean +} + +input UpdateStoryInput { + """ + id is the identifier of the Story used either when the Story was created via + the API or from Talk when it was lazily created. + """ + id: ID! + + """ + story contains the fields that should be updated. Any fields not specified + will not be changed. + """ + story: UpdateStory! + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +type UpdateStoryPayload { + """ + story is the Story that was possibly updated. + """ + story: Story + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +################## +## mergeStories +################## + +input MergeStoriesInput { + """ + sourceIDs are the list of Story ID's that should have their Comment's moved + onto the Story indicated by `destinationID`. The Stories indicated by the + `sourceIDs` field will be removed after the Comment's have been moved. + """ + sourceIDs: [ID!]! + + """ + destinationID is the ID of the Story where all the other "source" Stories will + have their Comment's moved onto. + """ + destinationID: ID! + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +type MergeStoriesPayload { + """ + story is the Story that all the source stories were merged into. + """ + story: Story + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +################## +## removeStory +################## + +input RemoveStoryInput { + """ + id is the identifier of the Story used either when the Story was created via + the API or from Talk when it was lazily created. + """ + id: ID! + + """ + includeComments when true will remove any Comment's that were left on the + Story. This option should be used rarely, instead preferring to updating the + Story URL and/or merging with the correct Story. + """ + includeComments: Boolean = false + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +type RemoveStoryPayload { + """ + story is the Story that was possibly removed. + """ + story: Story + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +################## +## scrapeStory +################## + +input ScrapeStoryInput { + """ + id is the identifier of the Story used either when the Story was created via + the API or from Talk when it was lazily created. + """ + id: ID! + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + +type ScrapeStoryPayload { + """ + story is the Story that was possibly scraped. + """ + story: Story + + """ + clientMutationId is required for Relay support. + """ + clientMutationId: String! +} + ################## ## Mutation ################## @@ -2355,12 +2601,12 @@ type Mutation { ): UpdateOIDCAuthIntegrationPayload @auth(roles: [ADMIN]) """ - deleteOIDCAuthIntegration will delete the specified OpenID Connect auth + removeOIDCAuthIntegration will remove the specified OpenID Connect auth integration. """ - deleteOIDCAuthIntegration( - input: DeleteOIDCAuthIntegrationInput! - ): DeleteOIDCAuthIntegrationPayload @auth(roles: [ADMIN]) + removeOIDCAuthIntegration( + input: RemoveOIDCAuthIntegrationInput! + ): RemoveOIDCAuthIntegrationPayload @auth(roles: [ADMIN]) """ createCommentReaction will create a Reaction authored by the current logged in @@ -2371,10 +2617,10 @@ type Mutation { ): CreateCommentReactionPayload @auth """ - deleteCommentReaction will delete a Reaction authored by the current logged in + removeCommentReaction will remove a Reaction authored by the current logged in User on a Comment if it exists. """ - deleteCommentReaction( + removeCommentReaction( input: CreateCommentReactionInput! ): CreateCommentReactionPayload @auth @@ -2387,10 +2633,10 @@ type Mutation { ): CreateCommentDontAgreePayload @auth """ - deleteCommentDontAgree will delete a DontAgree authored by the current logged in + removeCommentDontAgree will remove a DontAgree authored by the current logged in User on a Comment if it exists. """ - deleteCommentDontAgree( + removeCommentDontAgree( input: CreateCommentDontAgreeInput! ): CreateCommentDontAgreePayload @auth @@ -2402,11 +2648,42 @@ type Mutation { @auth """ - deleteCommentFlag will create a Flag authored by the current logged in User on + removeCommentFlag will create a Flag authored by the current logged in User on a given Comment. """ - deleteCommentFlag(input: DeleteCommentFlagInput!): DeleteCommentFlagPayload + removeCommentFlag(input: RemoveCommentFlagInput!): RemoveCommentFlagPayload @auth + + """ + createStory will create the provided Story. + """ + createStory(input: CreateStoryInput!): CreateStoryPayload + @auth(roles: [ADMIN]) + + """ + updateStory will update the given Story. + """ + updateStory(input: UpdateStoryInput!): UpdateStoryPayload + @auth(roles: [ADMIN]) + + """ + mergeStories will merge two stories together, merging their comment streams. + This operation is irreversible. + """ + mergeStories(input: MergeStoriesInput!): MergeStoriesPayload + @auth(roles: [ADMIN]) + + """ + removeStory will remove the given Story. + """ + removeStory(input: RemoveStoryInput!): RemoveStoryPayload + @auth(roles: [ADMIN]) + + """ + scrapeStory will scrape the given Story and update the scraped metadata. + """ + scrapeStory(input: ScrapeStoryInput!): ScrapeStoryPayload + @auth(roles: [ADMIN, MODERATOR]) } ################################################################################ diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 7529c2be7..44e5da226 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -5,7 +5,7 @@ import config, { Config } from "talk-common/config"; import getManagementSchema from "talk-server/graph/management/schema"; import { Schemas } from "talk-server/graph/schemas"; import getTenantSchema from "talk-server/graph/tenant/schema"; -import { createQueue } from "talk-server/services/queue"; +import { createQueue } from "talk-server/queue"; import TenantCache from "talk-server/services/tenant/cache"; import { createJWTSigningConfig } from "talk-server/services/jwt"; diff --git a/src/core/server/models/action.ts b/src/core/server/models/action.ts index bcc1c4fd3..6705bdb22 100644 --- a/src/core/server/models/action.ts +++ b/src/core/server/models/action.ts @@ -56,13 +56,54 @@ export type FLAG_REASON = | GQLCOMMENT_FLAG_REASON; export interface Action extends TenantResource { + /** + * id is the identifier for this specific Action. + */ readonly id: string; + + /** + * action_type is the type of Action that this represents. + */ action_type: ACTION_TYPE; + + /** + * item_type enables polymorphic behavior be allowing multiple item types + * to be represented in a single collection. + */ item_type: ACTION_ITEM_TYPE; + + /** + * item_id is the ID of the specific item that this Action is associated with. + */ item_id: string; + + /** + * reason is the reason or secondary grouping identifier for why this + * particular action was left. + */ reason?: FLAG_REASON; + + /** + * root_item_id represents the identifier for the item's associated item. In + * the case of a REACTION left on a Comment, this ID would be the Stories ID. + * In the case of a FLAG left on a User, this ID would be null. + */ + root_item_id?: string; + + /** + * user_id is the ID of the User that left this Action. In the event that the + * Action was left by Talk, it will be null. + */ user_id?: string; + + /** + * created_at is the date that this particular Action was created at. + */ created_at: Date; + + /** + * metadata is arbitrary information stored for this Action. + */ metadata?: Record; } @@ -248,36 +289,36 @@ export async function retrieveManyUserActionPresence( ); } -export type DeleteActionInput = Pick< +export type RemoveActionInput = Pick< Action, "action_type" | "item_type" | "item_id" | "reason" | "user_id" >; /** - * The result returned by `deleteAction`. + * The result returned by `removeAction`. */ -export interface DeletedActionResultObject { +export interface RemovedActionResultObject { /** * action is the action that was deleted. */ action?: Action; /** - * wasDeleted is true when the action that was supposed to be deleted was + * wasRemoved is true when the action that was supposed to be deleted was * actually deleted. */ - wasDeleted: boolean; + wasRemoved: boolean; } /** - * deleteAction will delete the action based on the form of the action rather + * removeAction will delete the action based on the form of the action rather * than a specific action by ID. */ -export async function deleteAction( +export async function removeAction( mongo: Db, tenantID: string, - input: DeleteActionInput -): Promise { + input: RemoveActionInput +): Promise { // Extract the filter parameters. const filter: FilterQuery = { tenant_id: tenantID, @@ -297,7 +338,7 @@ export async function deleteAction( const result = await collection(mongo).findOneAndDelete(filter); return { action: result.value, - wasDeleted: Boolean(result.ok && result.value), + wasRemoved: Boolean(result.ok && result.value), }; } @@ -455,6 +496,30 @@ function createEmptyActionCounts(): GQLActionCounts { }; } +export function mergeActionCounts( + actionCounts: EncodedActionCounts[] +): EncodedActionCounts { + const mergedActionCounts: EncodedActionCounts = {}; + + for (const counts of actionCounts) { + for (const [key, count] of Object.entries(counts)) { + if (key in mergedActionCounts) { + mergedActionCounts[key] += count; + } else { + mergedActionCounts[key] += 1; + } + } + } + + return mergedActionCounts; +} + +export function countTotalActionCounts( + actionCounts: EncodedActionCounts +): number { + return Object.values(actionCounts).reduce((total, count) => total + count, 0); +} + /** * decodeActionCounts will take the encoded action counts and decode them into * a useable format. @@ -469,7 +534,7 @@ export function decodeActionCounts( // Loop over all the encoded action counts to extract each of the action // counts as they are encoded. - Object.entries(encodedActionCounts).map(([key, count]) => { + Object.entries(encodedActionCounts).forEach(([key, count]) => { // Pull out the action type and the reason from the key. const { actionType, reason } = decodeActionCountKey(key); @@ -509,3 +574,43 @@ function incrementActionCounts( return actionCounts; } + +/** + * removeRootActions will remove all the Action's associated with a given root + * identifier. + */ +export async function removeRootActions( + mongo: Db, + tenantID: string, + rootItemID: string +) { + return collection(mongo).deleteMany({ + tenant_id: tenantID, + root_item_id: rootItemID, + }); +} + +/** + * mergeManyRootActions will update many Action `root_item_id'`s from one to + * another. + */ +export async function mergeManyRootActions( + mongo: Db, + tenantID: string, + newRootItemID: string, + oldRootItemIDs: string[] +) { + return collection(mongo).updateMany( + { + tenant_id: tenantID, + root_item_id: { + $in: oldRootItemIDs, + }, + }, + { + $set: { + root_item_id: newRootItemID, + }, + } + ); +} diff --git a/src/core/server/models/comment.ts b/src/core/server/models/comment.ts index 02a7ae8a6..e89018fb6 100644 --- a/src/core/server/models/comment.ts +++ b/src/core/server/models/comment.ts @@ -550,3 +550,43 @@ export async function updateCommentActionCounts( return result.value; } + +/** + * removeStoryComments will remove all comments associated with a particular + * Story. + */ +export async function removeStoryComments( + mongo: Db, + tenantID: string, + storyID: string +) { + // Delete all the comments written on a specific story. + return collection(mongo).deleteMany({ + tenant_id: tenantID, + story_id: storyID, + }); +} + +/** + * mergeManyCommentStories will update many comment's storyID's. + */ +export async function mergeManyCommentStories( + mongo: Db, + tenantID: string, + newStoryID: string, + oldStoryIDs: string[] +) { + return collection(mongo).updateMany( + { + tenant_id: tenantID, + story_id: { + $in: oldStoryIDs, + }, + }, + { + $set: { + story_id: newStoryID, + }, + } + ); +} diff --git a/src/core/server/models/story.ts b/src/core/server/models/story.ts index b6ac62d25..6a28b08ce 100644 --- a/src/core/server/models/story.ts +++ b/src/core/server/models/story.ts @@ -1,4 +1,4 @@ -import { Db } from "mongodb"; +import { Db, MongoError } from "mongodb"; import uuid from "uuid"; import { Omit } from "talk-common/types"; @@ -149,6 +149,37 @@ export async function updateCommentStatusCount( return result.value || null; } +/** + * mergeCommentStatusCount will merge an array of commentStatusCount's into one. + */ +export function mergeCommentStatusCount( + commentStatusCounts: CommentStatusCounts[] +): CommentStatusCounts { + const statusCounts = createEmptyCommentCounts(); + for (const commentCounts of commentStatusCounts) { + for (const status in commentCounts) { + if (!commentCounts.hasOwnProperty(status)) { + continue; + } + + // Because the CommentStatusCounts are not indexable, it should be accessed + // by walking the structure. + switch (status) { + case GQLCOMMENT_STATUS.ACCEPTED: + case GQLCOMMENT_STATUS.NONE: + case GQLCOMMENT_STATUS.PREMOD: + case GQLCOMMENT_STATUS.REJECTED: + case GQLCOMMENT_STATUS.SYSTEM_WITHHELD: + statusCounts[status] += commentCounts[status]; + break; + default: + throw new Error("unrecognized status"); + } + } + } + return statusCounts; +} + function createEmptyCommentCounts(): CommentStatusCounts { return { [GQLCOMMENT_STATUS.ACCEPTED]: 0, @@ -191,6 +222,46 @@ export async function findOrCreateStory( return upsertStory(db, tenantID, { url }); } +export type CreateStoryInput = Partial>; + +export async function createStory( + mongo: Db, + tenantID: string, + id: string, + url: string, + input: CreateStoryInput +) { + const now = new Date(); + + // Create the story. + const story: Story = { + ...input, + id, + url, + tenant_id: tenantID, + created_at: now, + action_counts: {}, + comment_counts: createEmptyCommentCounts(), + }; + + try { + // Insert the story into the database. + await collection(mongo).insertOne(story); + } catch (err) { + // Evaluate the error, if it is in regards to violating the unique index, + // then return a duplicate Story error. + if (err instanceof MongoError && err.code === 11000) { + // TODO: (wyattjoh) return better error + throw new Error("story with this url already exists"); + } + + throw err; + } + + // Return the created story. + return story; +} + export async function retrieveStoryByURL( db: Db, tenantID: string, @@ -235,7 +306,7 @@ export async function retrieveManyStoriesByURL( export type UpdateStoryInput = Omit< Partial, - "id" | "tenant_id" | "url" | "created_at" + "id" | "tenant_id" | "created_at" >; export async function updateStory( @@ -253,15 +324,26 @@ export async function updateStory( }, }; - const result = await collection(db).findOneAndUpdate( - { id, tenant_id: tenantID }, - update, - // False to return the updated document instead of the original - // document. - { returnOriginal: false } - ); + try { + const result = await collection(db).findOneAndUpdate( + { id, tenant_id: tenantID }, + update, + // False to return the updated document instead of the original + // document. + { returnOriginal: false } + ); - return result.value || null; + return result.value || null; + } catch (err) { + // Evaluate the error, if it is in regards to violating the unique index, + // then return a duplicate Story error. + if (input.url && err instanceof MongoError && err.code === 11000) { + // TODO: (wyattjoh) return better error + throw new Error("story with this url already exists"); + } + + throw err; + } } /** @@ -291,3 +373,58 @@ export async function updateStoryActionCounts( return result.value || null; } + +export async function removeStory(mongo: Db, tenantID: string, id: string) { + const result = await collection(mongo).findOneAndDelete({ + id, + tenant_id: tenantID, + }); + + return result.value || null; +} + +/** + * removeStories will remove the stories specified by the set of id's. + */ +export async function removeStories( + mongo: Db, + tenantID: string, + ids: string[] +) { + return collection(mongo).deleteMany({ + tenant_id: tenantID, + id: { + $in: ids, + }, + }); +} + +/** + * calculateTotalCommentCount will compute the total amount of comments left on + * an Asset by parsing the `CommentStatusCounts`. + */ +export function calculateTotalCommentCount( + commentCounts: CommentStatusCounts +): number { + let count = 0; + for (const status in commentCounts) { + if (!commentCounts.hasOwnProperty(status)) { + continue; + } + + // Because the CommentStatusCounts are not indexable, it should be accessed + // by walking the structure. + switch (status) { + case GQLCOMMENT_STATUS.ACCEPTED: + case GQLCOMMENT_STATUS.NONE: + case GQLCOMMENT_STATUS.PREMOD: + case GQLCOMMENT_STATUS.REJECTED: + case GQLCOMMENT_STATUS.SYSTEM_WITHHELD: + count += commentCounts[status]; + break; + default: + throw new Error("unrecognized status"); + } + } + return count; +} diff --git a/src/core/server/models/tenant.ts b/src/core/server/models/tenant.ts index 56a425f04..ceb9e2ad1 100644 --- a/src/core/server/models/tenant.ts +++ b/src/core/server/models/tenant.ts @@ -352,25 +352,25 @@ export async function updateTenantOIDCAuthIntegration( }; } -export interface DeleteTenantOIDCAuthIntegrationResultObject { +export interface RemoveTenantOIDCAuthIntegrationResultObject { tenant?: Tenant; integration?: Omit; - wasDeleted: boolean; + wasRemoved: boolean; } /** - * deleteTenantOIDCAuthIntegration will delete the specific OpenID Connect Auth + * removeTenantOIDCAuthIntegration will delete the specific OpenID Connect Auth * Integration on the Tenant. * * @param mongo MongoDB Database handle * @param id the id of the Tenant * @param oidcID the id of the OpenID Connect Auth Integration we're deleting */ -export async function deleteTenantOIDCAuthIntegration( +export async function removeTenantOIDCAuthIntegration( mongo: Db, id: string, oidcID: string -): Promise { +): Promise { const result = await collection(mongo).findOneAndUpdate( { id }, { @@ -384,7 +384,7 @@ export async function deleteTenantOIDCAuthIntegration( } ); if (!result.value) { - return { wasDeleted: false }; + return { wasRemoved: false }; } // Find the integration that we wanted to delete. @@ -394,7 +394,7 @@ export async function deleteTenantOIDCAuthIntegration( if (!integration) { // The integration was not in the original document, so we could not have // possibly deleted it! - return { wasDeleted: false }; + return { wasRemoved: false }; } // The integration was found, we should pull that integration out of the @@ -406,6 +406,6 @@ export async function deleteTenantOIDCAuthIntegration( return { tenant: result.value, integration, - wasDeleted: true, + wasRemoved: true, }; } diff --git a/src/core/server/services/queue/Task.ts b/src/core/server/queue/Task.ts similarity index 100% rename from src/core/server/services/queue/Task.ts rename to src/core/server/queue/Task.ts diff --git a/src/core/server/services/queue/index.ts b/src/core/server/queue/index.ts similarity index 90% rename from src/core/server/services/queue/index.ts rename to src/core/server/queue/index.ts index 91a2dc4c7..54b307074 100644 --- a/src/core/server/services/queue/index.ts +++ b/src/core/server/queue/index.ts @@ -2,15 +2,12 @@ import Queue from "bull"; import { Db } from "mongodb"; import { Config } from "talk-common/config"; -import Task from "talk-server/services/queue/Task"; -import { - createMailerTask, - Mailer, -} from "talk-server/services/queue/tasks/mailer"; +import Task from "talk-server/queue/Task"; +import { createMailerTask, Mailer } from "talk-server/queue/tasks/mailer"; import { createScraperTask, ScraperData, -} from "talk-server/services/queue/tasks/scraper"; +} from "talk-server/queue/tasks/scraper"; import { createRedisClient } from "talk-server/services/redis"; import TenantCache from "talk-server/services/tenant/cache"; diff --git a/src/core/server/services/queue/tasks/Task.ts b/src/core/server/queue/tasks/Task.ts similarity index 100% rename from src/core/server/services/queue/tasks/Task.ts rename to src/core/server/queue/tasks/Task.ts diff --git a/src/core/server/services/queue/tasks/mailer/content.ts b/src/core/server/queue/tasks/mailer/content.ts similarity index 100% rename from src/core/server/services/queue/tasks/mailer/content.ts rename to src/core/server/queue/tasks/mailer/content.ts diff --git a/src/core/server/services/queue/tasks/mailer/index.ts b/src/core/server/queue/tasks/mailer/index.ts similarity index 97% rename from src/core/server/services/queue/tasks/mailer/index.ts rename to src/core/server/queue/tasks/mailer/index.ts index 53b46c2e6..39a570fb4 100644 --- a/src/core/server/services/queue/tasks/mailer/index.ts +++ b/src/core/server/queue/tasks/mailer/index.ts @@ -6,8 +6,8 @@ import { createTransport } from "nodemailer"; import { Config } from "talk-common/config"; import logger from "talk-server/logger"; -import Task from "talk-server/services/queue/Task"; -import MailerContent from "talk-server/services/queue/tasks/mailer/content"; +import Task from "talk-server/queue/Task"; +import MailerContent from "talk-server/queue/tasks/mailer/content"; import TenantCache from "talk-server/services/tenant/cache"; import { TenantCacheAdapter } from "talk-server/services/tenant/cache/adapter"; diff --git a/src/core/server/services/queue/tasks/mailer/templates/forgot-password.html b/src/core/server/queue/tasks/mailer/templates/forgot-password.html similarity index 100% rename from src/core/server/services/queue/tasks/mailer/templates/forgot-password.html rename to src/core/server/queue/tasks/mailer/templates/forgot-password.html diff --git a/src/core/server/queue/tasks/scraper/index.ts b/src/core/server/queue/tasks/scraper/index.ts new file mode 100644 index 000000000..a6287f20b --- /dev/null +++ b/src/core/server/queue/tasks/scraper/index.ts @@ -0,0 +1,55 @@ +import Queue, { Job } from "bull"; +import { Db } from "mongodb"; + +import logger from "talk-server/logger"; +import Task from "talk-server/queue/Task"; +import { scrape } from "talk-server/services/stories/scraper"; + +const JOB_NAME = "scraper"; + +export interface ScrapeProcessorOptions { + mongo: Db; +} + +export interface ScraperData { + storyID: string; + storyURL: string; + tenantID: string; +} + +const createJobProcessor = ({ mongo }: ScrapeProcessorOptions) => async ( + job: Job +) => { + // Pull out the job data. + const { storyID, storyURL, tenantID } = job.data; + + const log = logger.child({ + job_id: job.id, + job_name: JOB_NAME, + story_id: storyID, + story_url: storyURL, + tenant_id: tenantID, + }); + + log.debug("starting to scrape the story"); + + try { + await scrape(mongo, tenantID, storyID, storyURL); + log.debug("scraped the story"); + } catch (err) { + log.error({ err }, "could not scrape the story"); + + throw err; + } +}; + +export function createScraperTask( + queue: Queue.QueueOptions, + options: ScrapeProcessorOptions +) { + return new Task({ + jobName: JOB_NAME, + jobProcessor: createJobProcessor(options), + queue, + }); +} diff --git a/src/core/server/services/comments/actions.ts b/src/core/server/services/comments/actions.ts index f152d1b10..b39c91d85 100644 --- a/src/core/server/services/comments/actions.ts +++ b/src/core/server/services/comments/actions.ts @@ -1,15 +1,16 @@ import { Db } from "mongodb"; +import { Omit } from "talk-common/types"; import { GQLCOMMENT_FLAG_REPORTED_REASON } from "talk-server/graph/tenant/schema/__generated__/types"; import { ACTION_ITEM_TYPE, ACTION_TYPE, CreateActionInput, createActions, - deleteAction, - DeleteActionInput, encodeActionCounts, invertEncodedActionCounts, + removeAction, + RemoveActionInput, } from "talk-server/models/action"; import { retrieveComment, @@ -20,11 +21,14 @@ import { updateStoryActionCounts } from "talk-server/models/story"; import { Tenant } from "talk-server/models/tenant"; import { User } from "talk-server/models/user"; +export type CreateAction = Omit & + Required>; + export async function addCommentActions( mongo: Db, tenant: Tenant, comment: Readonly, - inputs: CreateActionInput[] + inputs: CreateAction[] ): Promise> { // Create each of the actions, returning each of the action results. const results = await createActions(mongo, tenant.id, inputs); @@ -79,13 +83,19 @@ async function addCommentAction( throw new Error("comment not found"); } - return addCommentActions(mongo, tenant, comment, [input]); + // Store the story ID on the action as a story_id. + input.root_item_id = comment.story_id; + + // We have to perform a type assertion here because for some reason, the type + // coercion is not determining that because we filled in the `root_item_id` + // above, that at this point, it satisfies the CreateAction type. + return addCommentActions(mongo, tenant, comment, [input as CreateAction]); } export async function removeCommentAction( mongo: Db, tenant: Tenant, - input: DeleteActionInput + input: RemoveActionInput ): Promise> { // Get the Comment that we are leaving the Action on. const comment = await retrieveComment(mongo, tenant.id, input.item_id); @@ -95,8 +105,8 @@ export async function removeCommentAction( } // Create each of the actions, returning each of the action results. - const { wasDeleted, action } = await deleteAction(mongo, tenant.id, input); - if (wasDeleted) { + const { wasRemoved, action } = await removeAction(mongo, tenant.id, input); + if (wasRemoved) { // Compute the action counts, and invert them (because we're deleting an // action). const actionCounts = invertEncodedActionCounts(encodeActionCounts(action!)); @@ -145,13 +155,13 @@ export async function createReaction( }); } -export type DeleteCommentReaction = Pick; +export type RemoveCommentReaction = Pick; -export async function deleteReaction( +export async function removeReaction( mongo: Db, tenant: Tenant, author: User, - input: DeleteCommentReaction + input: RemoveCommentReaction ) { return removeCommentAction(mongo, tenant, { action_type: ACTION_TYPE.REACTION, @@ -177,13 +187,13 @@ export async function createDontAgree( }); } -export type DeleteCommentDontAgree = Pick; +export type RemoveCommentDontAgree = Pick; -export async function deleteDontAgree( +export async function removeDontAgree( mongo: Db, tenant: Tenant, author: User, - input: DeleteCommentDontAgree + input: RemoveCommentDontAgree ) { return removeCommentAction(mongo, tenant, { action_type: ACTION_TYPE.DONT_AGREE, @@ -212,13 +222,13 @@ export async function createFlag( }); } -export type DeleteCommentFlag = Pick; +export type RemoveCommentFlag = Pick; -export async function deleteFlag( +export async function removeFlag( mongo: Db, tenant: Tenant, author: User, - input: DeleteCommentFlag + input: RemoveCommentFlag ) { return removeCommentAction(mongo, tenant, { action_type: ACTION_TYPE.FLAG, diff --git a/src/core/server/services/comments/index.ts b/src/core/server/services/comments/index.ts index 89a06da38..c08221bbe 100644 --- a/src/core/server/services/comments/index.ts +++ b/src/core/server/services/comments/index.ts @@ -1,7 +1,7 @@ import { Db } from "mongodb"; import { Omit } from "talk-common/types"; -import { ACTION_ITEM_TYPE, CreateActionInput } from "talk-server/models/action"; +import { ACTION_ITEM_TYPE } from "talk-server/models/action"; import { createComment, CreateCommentInput, @@ -16,7 +16,10 @@ import { } from "talk-server/models/story"; import { Tenant } from "talk-server/models/tenant"; import { User } from "talk-server/models/user"; -import { addCommentActions } from "talk-server/services/comments/actions"; +import { + addCommentActions, + CreateAction, +} from "talk-server/services/comments/actions"; import { processForModeration } from "talk-server/services/comments/moderation"; import { Request } from "talk-server/types/express"; @@ -83,10 +86,13 @@ export async function create( // at the time, and we didn't want the repetitive nature of adding the // item_type each time, so this mapping function adds them! const inputs = actions.map( - (action): CreateActionInput => ({ + (action): CreateAction => ({ ...action, item_id: comment.id, item_type: ACTION_ITEM_TYPE.COMMENTS, + + // Store the Story ID on the action. + root_item_id: story.id, }) ); @@ -171,12 +177,15 @@ export async function edit( // at the time, and we didn't want the repetitive nature of adding the // item_type each time, so this mapping function adds them! const inputs = actions.map( - (action): CreateActionInput => ({ + (action): CreateAction => ({ ...action, // Strict null check seems to have failed here... Null checking was done // above where we errored if the comment was falsely. item_id: comment!.id, item_type: ACTION_ITEM_TYPE.COMMENTS, + + // Store the Story ID on the action. + root_item_id: story.id, }) ); diff --git a/src/core/server/services/stories/index.ts b/src/core/server/services/stories/index.ts index 79602b204..f30f15255 100644 --- a/src/core/server/services/stories/index.ts +++ b/src/core/server/services/stories/index.ts @@ -1,3 +1,4 @@ +import { zip } from "lodash"; import { Db } from "mongodb"; import { @@ -8,12 +9,36 @@ import { } from "talk-server/app/url"; import logger from "talk-server/logger"; import { + countTotalActionCounts, + mergeActionCounts, + mergeManyRootActions, + removeRootActions, +} from "talk-server/models/action"; +import { + mergeManyCommentStories, + removeStoryComments, +} from "talk-server/models/comment"; +import { + calculateTotalCommentCount, + createStory, + CreateStoryInput, findOrCreateStory, FindOrCreateStoryInput, + mergeCommentStatusCount, + removeStories, + removeStory, + retrieveManyStories, + retrieveStory, + Story, + updateCommentStatusCount, + updateStory, + updateStoryActionCounts, + UpdateStoryInput, } from "talk-server/models/story"; import { Tenant } from "talk-server/models/tenant"; -import Task from "talk-server/services/queue/Task"; -import { ScraperData } from "talk-server/services/queue/tasks/scraper"; +import Task from "talk-server/queue/Task"; +import { ScraperData } from "talk-server/queue/tasks/scraper"; +import { scrape } from "talk-server/services/stories/scraper"; export type FindOrCreateStory = FindOrCreateStoryInput; @@ -27,21 +52,24 @@ export async function findOrCreate( // to create the Asset. if (input.url && !isURLPermitted(tenant, input.url)) { logger.warn( - { storyURL: input.url, tenantDomains: tenant.domains }, - "provided story url was not in the list of permitted tenant domains" + { story_url: input.url, tenant_domains: tenant.domains }, + "provided story url was not in the list of permitted tenant domains, story not found" ); return null; } - // TODO: check to see if the tenant has enabled lazy story creation. + // TODO: check to see if the tenant has enabled lazy story creation, if they haven't, switch to find only. const story = await findOrCreateStory(db, tenant.id, input); if (!story) { return null; } - if (!story.scrapedAt) { - // If the scraper has not scraped this story, we need to scrape it now! + // TODO: check to see if the tenant has scraping enabled. + + if (!story.metadata && !story.scrapedAt) { + // If the scraper has not scraped this story, and we have no metadata, we + // need to scrape it now! await scraper.add({ storyID: story.id, storyURL: story.url, @@ -86,3 +114,235 @@ export function isURLPermitted( .map(domain => getOrigin(prefixSchemeIfRequired(originSecure, domain))) .some(origin => origin === targetOrigin); } + +export async function remove( + mongo: Db, + tenant: Tenant, + storyID: string, + includeComments: boolean = false +) { + // Create a logger for this function. + const log = logger.child({ + story_id: storyID, + include_comments: includeComments, + }); + + log.debug("starting to remove story"); + + // Get the story so we can see if there are associated comments. + const story = await retrieveStory(mongo, tenant.id, storyID); + if (!story) { + // No story was found! + log.warn("attempted to remove story that wasn't found"); + return null; + } + + if (includeComments) { + let removedCount: number | undefined; + + // Remove the actions associated with the comments we just removed. + ({ deletedCount: removedCount } = await removeRootActions( + mongo, + tenant.id, + story.id + )); + + log.debug( + { removed_actions: removedCount }, + "removed actions while deleting story" + ); + + // Remove the comments for the story. + ({ deletedCount: removedCount } = await removeStoryComments( + mongo, + tenant.id, + story.id + )); + + log.debug( + { removed_comments: removedCount }, + "removed comments while deleting story" + ); + } else if (calculateTotalCommentCount(story.comment_counts) > 0) { + log.warn( + "attempted to remove story that has linked comments without consent for deleting comments" + ); + + // TODO: (wyattjoh) improve error + throw new Error("asset has comments, cannot remove"); + } + + const removedStory = await removeStory(mongo, tenant.id, story.id); + if (!removedStory) { + // Story was already removed. + // TODO: evaluate use of transaction here. + return null; + } + + log.debug("removed story"); + + return removedStory; +} + +export type CreateStory = CreateStoryInput; + +export async function create( + mongo: Db, + tenant: Tenant, + storyID: string, + storyURL: string, + input: CreateStory +) { + // Ensure that the given URL is allowed. + if (!isURLPermitted(tenant, storyURL)) { + logger.warn( + { story_url: storyURL, tenant_domains: tenant.domains }, + "provided story url was not in the list of permitted tenant domains, story not created" + ); + return null; + } + + // Create the story in the database. + let newStory = await createStory(mongo, tenant.id, storyID, storyURL, input); + if (!input.metadata && !newStory.scrapedAt) { + // If the scraper has not scraped this story and story metadata was not + // provided, we need to scrape it now! + newStory = await scrape(mongo, tenant.id, newStory.id); + } + + return newStory; +} + +export type UpdateStory = UpdateStoryInput; + +export async function update( + mongo: Db, + tenant: Tenant, + storyID: string, + input: UpdateStory +) { + // Ensure that the given URL is allowed. + if (input.url && !isURLPermitted(tenant, input.url)) { + logger.warn( + { story_url: input.url, tenant_domains: tenant.domains }, + "provided story url was not in the list of permitted tenant domains, story not updated" + ); + return null; + } + + return updateStory(mongo, tenant.id, storyID, input); +} + +export async function merge( + mongo: Db, + tenant: Tenant, + destinationID: string, + sourceIDs: string[] +) { + // Create a logger for this operation. + const log = logger.child({ + destination_id: destinationID, + source_ids: sourceIDs, + }); + + if (sourceIDs.length === 0) { + log.warn("cannot merge from 0 stories"); + return null; + } + + // Get the stories referenced. + const storyIDs = [destinationID, ...sourceIDs]; + const stories = await retrieveManyStories(mongo, tenant.id, storyIDs); + + // Ensure that these are all defined. + if ( + zip(storyIDs, stories).some(([storyID, story]) => { + if (!story) { + log.warn( + { story_id: storyID }, + "story that was going to be merged was not found" + ); + return true; + } + + return false; + }) + ) { + return null; + } + + let updatedCount: number | undefined; + + // Move all the comment's from the source stories over to the destination + // story. + ({ modifiedCount: updatedCount } = await mergeManyCommentStories( + mongo, + tenant.id, + destinationID, + sourceIDs + )); + + log.debug( + { updated_comments: updatedCount }, + "updated comments while merging stories" + ); + + // Update all the action's that referenced the old story to reference the new + // story. + ({ modifiedCount: updatedCount } = await mergeManyRootActions( + mongo, + tenant.id, + destinationID, + sourceIDs + )); + + log.debug( + { updated_actions: updatedCount }, + "updated actions while merging stories" + ); + + // Merge the comment and action counts for all the source stories. + const [, ...sourceStories] = stories; + + let destinationStory = await updateCommentStatusCount( + mongo, + tenant.id, + destinationID, + mergeCommentStatusCount( + // We perform the type assertion here because above, we already verified + // that none of the stories are null. + (sourceStories as Story[]).map(({ comment_counts }) => comment_counts) + ) + ); + + const mergedActionCounts = mergeActionCounts( + // We perform the type assertion here because above, we already verified + // that none of the stories are null. + (sourceStories as Story[]).map(({ action_counts }) => action_counts) + ); + if (countTotalActionCounts(mergedActionCounts) > 0) { + destinationStory = await updateStoryActionCounts( + mongo, + tenant.id, + destinationID, + mergedActionCounts + ); + } + + if (!destinationStory) { + log.warn("destination story cannot be updated with new comment counts"); + return null; + } + + log.debug( + { comment_counts: destinationStory.comment_counts }, + "updated destination story with new comment counts" + ); + + const { deletedCount } = await removeStories(mongo, tenant.id, sourceIDs); + + log.debug({ deleted_stories: deletedCount }, "deleted source stories"); + + // Return the story that had the other stories merged into. + return destinationStory; +} diff --git a/src/core/server/services/queue/tasks/scraper/index.ts b/src/core/server/services/stories/scraper/index.ts similarity index 65% rename from src/core/server/services/queue/tasks/scraper/index.ts rename to src/core/server/services/stories/scraper/index.ts index 48d99a9f8..0a943e1c4 100644 --- a/src/core/server/services/queue/tasks/scraper/index.ts +++ b/src/core/server/services/stories/scraper/index.ts @@ -1,4 +1,3 @@ -import Queue, { Job } from "bull"; import Logger from "bunyan"; import cheerio from "cheerio"; import authorScraper from "metascraper-author"; @@ -10,87 +9,11 @@ import { Db } from "mongodb"; import { GQLStoryMetadata } from "talk-server/graph/tenant/schema/__generated__/types"; import logger from "talk-server/logger"; -import { updateStory } from "talk-server/models/story"; -import Task from "talk-server/services/queue/Task"; +import { retrieveStory, updateStory } from "talk-server/models/story"; + import { modifiedScraper } from "./rules/modified"; import { sectionScraper } from "./rules/section"; -const JOB_NAME = "scraper"; - -export interface ScrapeProcessorOptions { - mongo: Db; -} - -export interface ScraperData { - storyID: string; - storyURL: string; - tenantID: string; -} - -const createJobProcessor = (options: ScrapeProcessorOptions) => async ( - job: Job -) => { - // Pull out the job data. - const { storyID: id, storyURL: url, tenantID } = job.data; - - logger.debug( - { - job_id: job.id, - job_name: JOB_NAME, - story_id: id, - story_url: url, - tenant_id: tenantID, - }, - "starting to scrap the story" - ); - - // Get the metadata from the scraped html. - const metadata = await scraper.scrape(url); - if (!metadata) { - logger.error( - { - job_id: job.id, - job_name: JOB_NAME, - story_id: id, - story_url: url, - tenant_id: tenantID, - }, - "story at specified url not found, can not scrape" - ); - return; - } - - // Update the Story with the scraped details. - const story = await updateStory(options.mongo, tenantID, id, { - metadata, - scrapedAt: new Date(), - }); - if (!story) { - logger.error( - { - job_id: job.id, - job_name: JOB_NAME, - story_id: id, - story_url: url, - tenant_id: tenantID, - }, - "story at specified id not found, can not update with metadata" - ); - return; - } - - logger.debug( - { - job_id: job.id, - job_name: JOB_NAME, - story_id: story.id, - story_url: url, - tenant_id: tenantID, - }, - "scraped the story" - ); -}; - export type Rule = Record< string, Array< @@ -188,13 +111,37 @@ function createScraper() { export const scraper = createScraper(); -export function createScraperTask( - queue: Queue.QueueOptions, - options: ScrapeProcessorOptions +export async function scrape( + mongo: Db, + tenantID: string, + storyID: string, + storyURL?: string ) { - return new Task({ - jobName: JOB_NAME, - jobProcessor: createJobProcessor(options), - queue, + // If the URL wasn't provided, grab it from the database. + if (!storyURL) { + const retrievedStory = await retrieveStory(mongo, tenantID, storyID); + if (!retrievedStory) { + throw new Error("story at specified id not found"); + } + + // Update the story URL. + storyURL = retrievedStory.url; + } + + // Get the metadata from the scraped html. + const metadata = await scraper.scrape(storyURL); + if (!metadata) { + throw new Error("story at specified url not found"); + } + + // Update the Story with the scraped details. + const story = await updateStory(mongo, tenantID, storyID, { + metadata, + scrapedAt: new Date(), }); + if (!story) { + throw new Error("story at specified id not found"); + } + + return story; } diff --git a/src/core/server/services/queue/tasks/scraper/rules/modified.ts b/src/core/server/services/stories/scraper/rules/modified.ts similarity index 100% rename from src/core/server/services/queue/tasks/scraper/rules/modified.ts rename to src/core/server/services/stories/scraper/rules/modified.ts diff --git a/src/core/server/services/queue/tasks/scraper/rules/section.ts b/src/core/server/services/stories/scraper/rules/section.ts similarity index 100% rename from src/core/server/services/queue/tasks/scraper/rules/section.ts rename to src/core/server/services/stories/scraper/rules/section.ts diff --git a/src/core/server/services/tenant/index.ts b/src/core/server/services/tenant/index.ts index 0fa525616..0dfba01bb 100644 --- a/src/core/server/services/tenant/index.ts +++ b/src/core/server/services/tenant/index.ts @@ -11,8 +11,8 @@ import { createTenant, CreateTenantInput, createTenantOIDCAuthIntegration, - deleteTenantOIDCAuthIntegration, regenerateTenantSSOKey, + removeTenantOIDCAuthIntegration, Tenant, updateTenant, updateTenantOIDCAuthIntegration, @@ -176,7 +176,7 @@ export async function updateOIDCAuthIntegration( return result; } -export async function deleteOIDCAuthIntegration( +export async function removeOIDCAuthIntegration( mongo: Db, redis: Redis, cache: TenantCache, @@ -184,12 +184,12 @@ export async function deleteOIDCAuthIntegration( oidcID: string ) { // Delete the integration. By default, the integration is disabled. - const result = await deleteTenantOIDCAuthIntegration( + const result = await removeTenantOIDCAuthIntegration( mongo, tenant.id, oidcID ); - if (!result.wasDeleted || !result.tenant) { + if (!result.wasRemoved || !result.tenant) { return null; }