mirror of
https://github.com/wassname/talk.git
synced 2026-08-01 13:00:55 +08:00
Wyatt Johnson
@@ -1,183 +1 @@
|
||||
import Logger from "bunyan";
|
||||
import cheerio from "cheerio";
|
||||
import authorScraper from "metascraper-author";
|
||||
import dateScraper from "metascraper-date";
|
||||
import descriptionScraper from "metascraper-description";
|
||||
import imageScraper from "metascraper-image";
|
||||
import titleScraper from "metascraper-title";
|
||||
import { Db } from "mongodb";
|
||||
import fetch, { RequestInit } from "node-fetch";
|
||||
import ProxyAgent from "proxy-agent";
|
||||
|
||||
import { version } from "coral-common/version";
|
||||
import { GQLStoryMetadata } from "coral-server/graph/tenant/schema/__generated__/types";
|
||||
import logger from "coral-server/logger";
|
||||
import { retrieveStory, updateStory } from "coral-server/models/story";
|
||||
import { retrieveTenant } from "coral-server/models/tenant";
|
||||
|
||||
import { modifiedScraper } from "./rules/modified";
|
||||
import { sectionScraper } from "./rules/section";
|
||||
|
||||
export type Rule = Record<
|
||||
string,
|
||||
Array<
|
||||
(options: { htmlDom: CheerioSelector; url: string }) => string | undefined
|
||||
>
|
||||
>;
|
||||
|
||||
class Scraper {
|
||||
private rules: Rule[];
|
||||
private log: Logger;
|
||||
|
||||
constructor(rules: Rule[]) {
|
||||
this.rules = rules;
|
||||
this.log = logger.child({ taskName: "scraper" }, true);
|
||||
}
|
||||
|
||||
public async scrape(
|
||||
url: string,
|
||||
proxyURL?: string
|
||||
): Promise<GQLStoryMetadata | null> {
|
||||
// Grab the page HTML.
|
||||
|
||||
const log = this.log.child({ storyURL: url }, true);
|
||||
|
||||
const options: RequestInit = {
|
||||
headers: {
|
||||
"User-Agent": `Talk Scraper/${version}`,
|
||||
},
|
||||
};
|
||||
if (proxyURL) {
|
||||
// Force the type here because there's a slight mismatch.
|
||||
options.agent = (new ProxyAgent(
|
||||
proxyURL
|
||||
) as unknown) as RequestInit["agent"];
|
||||
log.debug("using proxy for scrape");
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
log.debug("starting scrape of Story");
|
||||
|
||||
const res = await fetch(url, options);
|
||||
if (res.status !== 200) {
|
||||
log.warn(
|
||||
{ statusCode: res.status },
|
||||
"scrape failed with non-200 status code"
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await res.text();
|
||||
|
||||
log.debug({ timeElapsed: Date.now() - start }, "scrape complete");
|
||||
|
||||
// Load the DOM.
|
||||
const htmlDom = cheerio.load(html);
|
||||
|
||||
log.debug("parsed html");
|
||||
|
||||
// Gather the results by evaluating each of the rules.
|
||||
const metadata: Record<string, string | undefined> = {};
|
||||
|
||||
for (const rule of this.rules) {
|
||||
for (const property in rule) {
|
||||
if (!rule.hasOwnProperty(property)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Proceed through each of the properties and try to find the mapped
|
||||
// properties.
|
||||
for (const getter of rule[property]) {
|
||||
const value = getter({ htmlDom, url });
|
||||
if (value && value.length > 0) {
|
||||
metadata[property] = value;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("extracted metadata");
|
||||
|
||||
return {
|
||||
title: metadata.title || undefined,
|
||||
description: metadata.description || undefined,
|
||||
image: metadata.image ? metadata.image : undefined,
|
||||
author: metadata.author || undefined,
|
||||
publishedAt: metadata.date ? new Date(metadata.date) : undefined,
|
||||
modifiedAt: metadata.modified ? new Date(metadata.modified) : undefined,
|
||||
section: metadata.section || undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* createScraper will create a scraper that will utilize the rules defined to
|
||||
* scrape metadata from the target page.
|
||||
*/
|
||||
function createScraper() {
|
||||
return new Scraper([
|
||||
authorScraper(),
|
||||
dateScraper(),
|
||||
descriptionScraper(),
|
||||
imageScraper(),
|
||||
titleScraper(),
|
||||
modifiedScraper(),
|
||||
sectionScraper(),
|
||||
]);
|
||||
}
|
||||
|
||||
export const scraper = createScraper();
|
||||
|
||||
export async function scrape(
|
||||
mongo: Db,
|
||||
tenantID: string,
|
||||
storyID: string,
|
||||
storyURL?: string
|
||||
) {
|
||||
// Grab the Tenant.
|
||||
const tenant = await retrieveTenant(mongo, tenantID);
|
||||
if (!tenant) {
|
||||
throw new Error("tenant not found");
|
||||
}
|
||||
|
||||
// If the URL wasn't provided, grab it from the database.
|
||||
if (!storyURL) {
|
||||
const retrievedStory = await retrieveStory(mongo, tenantID, storyID);
|
||||
if (!retrievedStory) {
|
||||
throw new Error("story at specified id not found");
|
||||
}
|
||||
|
||||
// Update the story URL.
|
||||
storyURL = retrievedStory.url;
|
||||
}
|
||||
|
||||
// Get the metadata from the scraped html.
|
||||
const metadata = await scraper.scrape(
|
||||
storyURL,
|
||||
tenant.stories.scraping.proxyURL
|
||||
);
|
||||
if (!metadata) {
|
||||
throw new Error("story at specified url not found");
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Update the Story with the scraped details.
|
||||
const story = await updateStory(
|
||||
mongo,
|
||||
tenantID,
|
||||
storyID,
|
||||
{
|
||||
metadata,
|
||||
scrapedAt: now,
|
||||
},
|
||||
now
|
||||
);
|
||||
if (!story) {
|
||||
throw new Error("story at specified id not found");
|
||||
}
|
||||
|
||||
return story;
|
||||
}
|
||||
export * from "./scraper";
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { $jsonld } from "@metascraper/helpers";
|
||||
import { $jsonld, date, toRule } from "@metascraper/helpers";
|
||||
import { Rules } from "metascraper";
|
||||
|
||||
import { wrap } from "./helpers";
|
||||
const toDate = toRule(date);
|
||||
|
||||
export const modifiedScraper = (): Rules => ({
|
||||
modified: [
|
||||
// From: http://ogp.me/#type_article
|
||||
wrap($jsonld("dateModified")),
|
||||
wrap($ => $('meta[property="article:modified"]').attr("content")),
|
||||
toDate($jsonld("dateModified")),
|
||||
toDate($ => $('meta[property="article:modified"]').attr("content")),
|
||||
],
|
||||
});
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { $jsonld, date, toRule } from "@metascraper/helpers";
|
||||
import { Rules } from "metascraper";
|
||||
|
||||
const toDate = toRule(date);
|
||||
|
||||
export const publishedScraper = (): Rules => ({
|
||||
published: [
|
||||
// From: http://ogp.me/#type_article
|
||||
toDate($jsonld("datePublished")),
|
||||
toDate($jsonld("dateCreated")),
|
||||
toDate($ => $('meta[property*="published_time" i]').attr("content")),
|
||||
toDate($ => $('meta[property*="release_date" i]').attr("content")),
|
||||
toDate($ => $('meta[name="date" i]').attr("content")),
|
||||
toDate($ => $('[itemprop="datepublished" i]').attr("content")),
|
||||
toDate($ => $('[itemprop*="date" i]').attr("content")),
|
||||
toDate($ => $('time[itemprop*="date" i]').attr("datetime")),
|
||||
toDate($ => $("time[datetime]").attr("datetime")),
|
||||
toDate($ => $("time[datetime][pubdate]").attr("datetime")),
|
||||
toDate($ => $('meta[name*="dc.date" i]').attr("content")),
|
||||
toDate($ => $('meta[name*="dc.date.issued" i]').attr("content")),
|
||||
toDate($ => $('meta[name*="dc.date.created" i]').attr("content")),
|
||||
toDate($ => $('meta[name*="dcterms.date" i]').attr("content")),
|
||||
toDate($ => $('[property*="dc:date" i]').attr("content")),
|
||||
toDate($ => $('[property*="dc:created" i]').attr("content")),
|
||||
],
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { scraper } from "./scraper";
|
||||
|
||||
describe("Scraper", () => {
|
||||
it("parses the JSON-LD data correctly", async () => {
|
||||
const html = `
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@type": "Article",
|
||||
"@id": "https://coralproject.net/blog/working-with-user-stories-keeping-commenters-and-moderators-at-the-center-of-what-we-build/",
|
||||
"author": {
|
||||
"@type": "Person",
|
||||
"name": "sam"
|
||||
},
|
||||
"headline": "Working with User Stories: Keeping commenters and moderators at the center of what we build",
|
||||
"description": "We believe that the comments section can be a place where diverse voices come together to share opinions and experiences.",
|
||||
"datePublished": "2019-09-04T15:43:35+00:00",
|
||||
"dateModified": "2019-09-06T06:14:29+00:00",
|
||||
"image": {
|
||||
"@type": "ImageObject",
|
||||
"url": "https://coralproject.net/wp-content/uploads/2019/09/blog-hero.png",
|
||||
"width": 1440,
|
||||
"height": 1024
|
||||
},
|
||||
"articleSection": "Comments,Design,Moderation,Useful"
|
||||
}
|
||||
</script>
|
||||
`;
|
||||
|
||||
expect(scraper.parse("", html)).toEqual({
|
||||
author: "sam",
|
||||
description:
|
||||
"We believe that the comments section can be a place where diverse voices come together to share opinions and experiences.",
|
||||
image:
|
||||
"https://coralproject.net/wp-content/uploads/2019/09/blog-hero.png",
|
||||
modifiedAt: new Date("2019-09-06T06:14:29+00:00"),
|
||||
publishedAt: new Date("2019-09-04T15:43:35+00:00"),
|
||||
section: "Comments,Design,Moderation,Useful",
|
||||
title:
|
||||
"Working with User Stories: Keeping commenters and moderators at the center of what we build",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
import Logger from "bunyan";
|
||||
import cheerio from "cheerio";
|
||||
import authorScraper from "metascraper-author";
|
||||
import descriptionScraper from "metascraper-description";
|
||||
import imageScraper from "metascraper-image";
|
||||
import titleScraper from "metascraper-title";
|
||||
import { Db } from "mongodb";
|
||||
import fetch, { RequestInit } from "node-fetch";
|
||||
import ProxyAgent from "proxy-agent";
|
||||
|
||||
import { version } from "coral-common/version";
|
||||
import logger from "coral-server/logger";
|
||||
import { retrieveStory, updateStory } from "coral-server/models/story";
|
||||
import { retrieveTenant } from "coral-server/models/tenant";
|
||||
|
||||
import { GQLStoryMetadata } from "coral-server/graph/tenant/schema/__generated__/types";
|
||||
|
||||
import { modifiedScraper } from "./rules/modified";
|
||||
import { publishedScraper } from "./rules/published";
|
||||
import { sectionScraper } from "./rules/section";
|
||||
|
||||
export type Rule = Record<
|
||||
string,
|
||||
Array<
|
||||
(options: { htmlDom: CheerioSelector; url: string }) => string | undefined
|
||||
>
|
||||
>;
|
||||
|
||||
class Scraper {
|
||||
private rules: Rule[];
|
||||
private log: Logger;
|
||||
|
||||
constructor(rules: Rule[]) {
|
||||
this.rules = rules;
|
||||
this.log = logger.child({ taskName: "scraper" }, true);
|
||||
}
|
||||
|
||||
public parse(url: string, html: string): GQLStoryMetadata {
|
||||
const log = this.log.child({ storyURL: url }, true);
|
||||
|
||||
// Load the DOM.
|
||||
const htmlDom = cheerio.load(html);
|
||||
|
||||
log.debug("parsed html");
|
||||
|
||||
// Gather the results by evaluating each of the rules.
|
||||
const metadata: Record<string, string | undefined> = {};
|
||||
|
||||
for (const rule of this.rules) {
|
||||
for (const property in rule) {
|
||||
if (!rule.hasOwnProperty(property)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Proceed through each of the properties and try to find the mapped
|
||||
// properties.
|
||||
for (const getter of rule[property]) {
|
||||
const value = getter({ htmlDom, url });
|
||||
if (value && value.length > 0) {
|
||||
metadata[property] = value;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.debug("extracted metadata");
|
||||
|
||||
return {
|
||||
title: metadata.title || undefined,
|
||||
description: metadata.description || undefined,
|
||||
image: metadata.image ? metadata.image : undefined,
|
||||
author: metadata.author || undefined,
|
||||
publishedAt: metadata.published
|
||||
? new Date(metadata.published)
|
||||
: undefined,
|
||||
modifiedAt: metadata.modified ? new Date(metadata.modified) : undefined,
|
||||
section: metadata.section || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
public async download(url: string, proxyURL?: string) {
|
||||
const log = this.log.child({ storyURL: url }, true);
|
||||
|
||||
const options: RequestInit = {
|
||||
headers: {
|
||||
"User-Agent": `Talk Scraper/${version}`,
|
||||
},
|
||||
};
|
||||
if (proxyURL) {
|
||||
// Force the type here because there's a slight mismatch.
|
||||
options.agent = (new ProxyAgent(
|
||||
proxyURL
|
||||
) as unknown) as RequestInit["agent"];
|
||||
log.debug("using proxy for scrape");
|
||||
}
|
||||
|
||||
const start = Date.now();
|
||||
log.debug("starting scrape of Story");
|
||||
|
||||
const res = await fetch(url, options);
|
||||
if (!res.ok || res.status !== 200) {
|
||||
log.warn(
|
||||
{ statusCode: res.status, statusText: res.statusText },
|
||||
"scrape failed with non-200 status code"
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await res.text();
|
||||
|
||||
log.debug({ timeElapsed: Date.now() - start }, "scrape complete");
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
public async scrape(
|
||||
url: string,
|
||||
proxyURL?: string
|
||||
): Promise<GQLStoryMetadata | null> {
|
||||
const html = await this.download(url, proxyURL);
|
||||
if (!html) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.parse(url, html);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* createScraper will create a scraper that will utilize the rules defined to
|
||||
* scrape metadata from the target page.
|
||||
*/
|
||||
function createScraper() {
|
||||
return new Scraper([
|
||||
authorScraper(),
|
||||
publishedScraper(),
|
||||
descriptionScraper(),
|
||||
imageScraper(),
|
||||
titleScraper(),
|
||||
modifiedScraper(),
|
||||
sectionScraper(),
|
||||
]);
|
||||
}
|
||||
|
||||
export const scraper = createScraper();
|
||||
|
||||
export async function scrape(
|
||||
mongo: Db,
|
||||
tenantID: string,
|
||||
storyID: string,
|
||||
storyURL?: string
|
||||
) {
|
||||
// Grab the Tenant.
|
||||
const tenant = await retrieveTenant(mongo, tenantID);
|
||||
if (!tenant) {
|
||||
throw new Error("tenant not found");
|
||||
}
|
||||
|
||||
// If the URL wasn't provided, grab it from the database.
|
||||
if (!storyURL) {
|
||||
const retrievedStory = await retrieveStory(mongo, tenantID, storyID);
|
||||
if (!retrievedStory) {
|
||||
throw new Error("story at specified id not found");
|
||||
}
|
||||
|
||||
// Update the story URL.
|
||||
storyURL = retrievedStory.url;
|
||||
}
|
||||
|
||||
// Get the metadata from the scraped html.
|
||||
const metadata = await scraper.scrape(
|
||||
storyURL,
|
||||
tenant.stories.scraping.proxyURL
|
||||
);
|
||||
if (!metadata) {
|
||||
throw new Error("story at specified url not found");
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
|
||||
// Update the Story with the scraped details.
|
||||
const story = await updateStory(
|
||||
mongo,
|
||||
tenantID,
|
||||
storyID,
|
||||
{
|
||||
metadata,
|
||||
scrapedAt: now,
|
||||
},
|
||||
now
|
||||
);
|
||||
if (!story) {
|
||||
throw new Error("story at specified id not found");
|
||||
}
|
||||
|
||||
return story;
|
||||
}
|
||||
Reference in New Issue
Block a user