[next] Story Improvements (#2054)

* feat: improved logs

* feat: improved scraper

- added scraper debug graph call

* feat: improved story closing

* fix: fixed tests

src/core/server/graph/tenant/loaders/stories.ts

@@ -1,11 +1,13 @@
 import DataLoader from "dataloader";
 
 import TenantContext from "talk-server/graph/tenant/context";
+import { GQLStoryMetadata } from "talk-server/graph/tenant/schema/__generated__/types";
 import {
   FindOrCreateStoryInput,
   retrieveManyStories,
   Story,
 } from "talk-server/models/story";
+import { scraper } from "talk-server/services/queue/tasks/scraper";
 import { findOrCreate } from "talk-server/services/stories";
 
 export default (ctx: TenantContext) => ({
@@ -14,4 +16,7 @@ export default (ctx: TenantContext) => ({
   story: new DataLoader<string, Story | null>(ids =>
     retrieveManyStories(ctx.mongo, ctx.tenant.id, ids)
   ),
+  debugScrapeMetadata: new DataLoader<string, GQLStoryMetadata | null>(urls =>
+    Promise.all(urls.map(url => scraper.scrape(url)))
+  ),
 });

src/core/server/graph/tenant/resolvers/query.ts

@@ -8,6 +8,8 @@ const Query: GQLQueryTypeResolver<void> = {
   me: (source, args, ctx) => ctx.user,
   discoverOIDCConfiguration: (source, { issuer }, ctx) =>
     ctx.loaders.Auth.discoverOIDCConfiguration.load(issuer),
+  debugScrapeStoryMetadata: (source, { url }, ctx) =>
+    ctx.loaders.Stories.debugScrapeMetadata.load(url),
 };
 
 export default Query;

src/core/server/graph/tenant/resolvers/story.ts

@@ -1,3 +1,5 @@
+import { DateTime } from "luxon";
+
 import { GQLStoryTypeResolver } from "talk-server/graph/tenant/schema/__generated__/types";
 import { decodeActionCounts } from "talk-server/models/action";
 import { Story } from "talk-server/models/story";
@@ -5,8 +7,20 @@ import { Story } from "talk-server/models/story";
 const Story: GQLStoryTypeResolver<Story> = {
   comments: (story, input, ctx) =>
     ctx.loaders.Comments.forStory(story.id, input),
-  // TODO: implement this.
   isClosed: () => false,
+  closedAt: (story, input, ctx) => {
+    if (story.closedAt) {
+      return story.closedAt;
+    }
+
+    if (ctx.tenant.autoCloseStream && ctx.tenant.closedTimeout) {
+      return DateTime.fromJSDate(story.created_at)
+        .plus(ctx.tenant.closedTimeout)
+        .toJSDate();
+    }
+
+    return null;
+  },
   actionCounts: story => decodeActionCounts(story.action_counts),
   commentCounts: story => story.comment_counts,
 };

src/core/server/graph/tenant/schema/schema.graphql

@@ -1211,6 +1211,49 @@ enum COMMENT_SORT {
   RESPECT_DESC
 }
 
+"""
+StoryMetadata stores all the metadata that is scraped using the scraping tools
+inside Talk. Talk utilizes [metascraper](https://metascraper.js.org/) which uses
+a variety of filters derived from [The Open Graph Protocol](https://ogp.me/) to
+scan for metadata on the page.
+"""
+type StoryMetadata {
+  """
+  title stores the scraped title from the Story page.
+  """
+  title: String
+
+  """
+  author stores the scraped author from the Story page.
+  """
+  author: String
+
+  """
+  description stores the scraped description from the Story page.
+  """
+  description: String
+
+  """
+  image stores the scraped image from the Story page.
+  """
+  image: String
+
+  """
+  publishedAt stores the scraped publication date from the Story page.
+  """
+  publishedAt: Time
+
+  """
+  modifiedAt stores the scraped modified date from the Story page.
+  """
+  modifiedAt: Time
+
+  """
+  section stores the scraped section from the Story page.
+  """
+  section: String
+}
+
 """
 Story is an Article or Page where Comments are written on by Users.
 """
@@ -1226,9 +1269,14 @@ type Story {
   url: String!
 
   """
-  title is the title of the scraped Story.
+  metadata stores the scraped metadata from the Story page.
   """
-  title: String
+  metadata: StoryMetadata
+
+  """
+  scrapedAt is the Time that the Story had it's metadata scraped at.
+  """
+  scrapedAt: Time
 
   """
   comments are the comments on the Story.
@@ -1245,11 +1293,6 @@ type Story {
   """
   actionCounts: ActionCounts! @auth(roles: [ADMIN, MODERATOR])
 
-  """
-  author is the authors listed in the meta tags for the Story.
-  """
-  author: String
-
   """
   closedAt is the Time that the Story is closed for commenting.
   """
@@ -1338,6 +1381,13 @@ type Query {
   """
   discoverOIDCConfiguration(issuer: String!): DiscoveredOIDCConfiguration
     @auth(roles: [ADMIN])
+
+  """
+  debugScrapeStoryMetadata will return the information that Talk was able to
+  scrape from the given URL. No data will be saved related to the Story based
+  on this scrape.
+  """
+  debugScrapeStoryMetadata(url: String!): StoryMetadata @auth(roles: [ADMIN])
 }
 
 ################################################################################
@@ -1679,7 +1729,7 @@ SettingsInput is the partial type of the Settings type for performing mutations.
 """
 input SettingsInput {
   """
-  domains will return a given list of whitelisted domains.
+  domains will return a given list of domains that hosts Stories.
   """
   domains: [String!]