[CORL-735] Scraper support for JSON Linked Data (#2671)

* feat: add support for application/json+ld parsing

- upgraded scraper packages to use new jsonld parser
- waiting on https://github.com/microlinkhq/metascraper/pull/225
  for merging
- fixes #2670

* chore: metascraper upgrades
This commit is contained in:
Wyatt Johnson
2019-10-24 16:08:15 +00:00
committed by GitHub
parent ac96e5e312
commit 5714f441bb
6 changed files with 601 additions and 149 deletions
@@ -0,0 +1,4 @@
import { Rule, Ruler } from "metascraper";
export const wrap = (rule: Rule): Ruler => ({ htmlDom, url }) =>
rule(htmlDom, url);
@@ -1,8 +1,12 @@
import { $jsonld } from "@metascraper/helpers";
import { Rules } from "metascraper";
import { wrap } from "./helpers";
export const modifiedScraper = (): Rules => ({
modified: [
// From: http://ogp.me/#type_article
({ htmlDom: $ }) => $('meta[property="article:modified"]').attr("content"),
wrap($jsonld("dateModified")),
wrap($ => $('meta[property="article:modified"]').attr("content")),
],
});
@@ -1,8 +1,12 @@
import { $jsonld } from "@metascraper/helpers";
import { Rules } from "metascraper";
import { wrap } from "./helpers";
export const sectionScraper = (): Rules => ({
section: [
// From: http://ogp.me/#type_article
({ htmlDom: $ }) => $('meta[property="article:section"]').attr("content"),
wrap($jsonld("articleSection")),
wrap($ => $('meta[property="article:section"]').attr("content")),
],
});