diff --git a/bin/cli-assets b/bin/cli-assets index 8242a02d6..d626ba094 100755 --- a/bin/cli-assets +++ b/bin/cli-assets @@ -219,6 +219,28 @@ async function rewrite(search, replace, options) { } } +/** + * debugAssetURL will scrape the given URL and print it's scraped values. + */ +async function debugAssetURL(url) { + try { + const meta = await scraper.scrape(url); + + const table = new Table({ head: ['Property', 'Value'] }); + + for (const [property, value] of Object.entries(meta)) { + table.push([property, value]); + } + + console.log(table.toString()); + + util.shutdown(0); + } catch (err) { + console.error(err); + util.shutdown(1); + } +} + //============================================================================== // Setting up the program command line arguments. //============================================================================== @@ -234,6 +256,13 @@ program .description('list all the assets in the database') .action(listAssets); +program + .command('debug [url]') + .description( + 'prints the scraped metadata that would be added to the given asset' + ) + .action(debugAssetURL); + program .command('refresh [age]') .description('queues the assets that exceed the age requested') diff --git a/docs/_config.yml b/docs/_config.yml index 041054bda..0931ecf5b 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -104,6 +104,8 @@ sidebar: url: /integrating/authentication/ - title: Asset Management url: /integrating/asset-management/ + - title: Asset Scraping + url: /integrating/asset-scraping/ - title: Configuring the Comment Stream url: /integrating/configuring-comment-stream/ - title: Configuring the Admin diff --git a/docs/source/integrating/asset-scraping.md b/docs/source/integrating/asset-scraping.md new file mode 100644 index 000000000..5d524f062 --- /dev/null +++ b/docs/source/integrating/asset-scraping.md @@ -0,0 +1,44 @@ +--- +title: Asset Scraping +permalink: /integrating/asset-scraping/ +--- + +By default, Assets in Talk have their metadata scraped when they are loaded. +This provides the easiest way for newsrooms to integrate their CMS's into Talk +in a simple way. We use the following +[meta tags](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta) on +the target pages that allow us to extract some properties. + +| Asset Property | Selector | +|--------------------|----------| +| `title` | See [`metascraper-title`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-title/index.js) | +| `description` | See [`metascraper-description`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-description/index.js) | +| `image` | See [`metascraper-image`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-image/index.js) | +| `author` | See [`metascraper-author`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-author/index.js) | +| `publication_date` | See [`metascraper-date`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-date/index.js) | +| `modified_date` | `meta[property="article:modified"]` | +| `section` | `meta[property="article:section"]` | + +You can use the `./bin/cli assets debug ` to print the scraped metadata +from that URL. For example: + +```bash + $ ./bin/cli assets debug https://www.washingtonpost.com/technology/2018/10/30/apple-event-october-ipad-pro-macbook-air/ +┌──────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Property │ Value │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ title │ Apple redesigns the iPad Pro, breathes new life in the MacBook Air │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ description │ Apple is unveiling new iPads and MacBooks at an event in New York starting at 10 a.m. Fowler is there and will report in with the news and hands-on analysis throughout the day. │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ image │ https://www.washingtonpost.com/resizer/JAwNQE2alL2JjiWrbXeJ46wZHqA=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/G5TWBFW4LAI6RC5MX7QB7TODUY.jpg │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ author │ Geoffrey A. Fowler │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ publication_date │ 2018-10-30T10:40:00.000Z │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ modified_date │ │ +├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ section │ │ +└──────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` \ No newline at end of file diff --git a/jobs/scraper/index.js b/jobs/scraper/index.js index 77d429042..ef64a2e25 100644 --- a/jobs/scraper/index.js +++ b/jobs/scraper/index.js @@ -1,61 +1,9 @@ const Asset = require('../../models/asset'); -const scraper = require('../../services/scraper'); +const scraper = require('../../services/scraper/scraper'); const Assets = require('../../services/assets'); const { createLogger } = require('../../services/logging'); const logger = createLogger('jobs:scraper'); -const fetch = require('node-fetch'); const { merge } = require('lodash'); -const { version } = require('../../package.json'); -const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config'); -const ProxyAgent = require('proxy-agent'); - -// Load the scraper with the rules. -const metascraper = require('metascraper').load([ - require('metascraper-title')(), - require('metascraper-description')(), - require('metascraper-image')(), - require('metascraper-author')(), - require('metascraper-date')(), - require('./rules/modified')(), - require('./rules/section')(), -]); - -let customHeaders = {}; -try { - customHeaders = JSON.parse(SCRAPER_HEADERS); -} catch (err) { - console.error('Cannot parse TALK_SCRAPER_HEADERS'); - throw err; -} - -// Parse the headers to be added to the scraper. -const headers = merge( - { - 'User-Agent': `Coral-Talk/${version}`, - }, - customHeaders -); - -// Add proxy configuration if exists. -const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null; - -/** - * Scrapes the given asset for metadata. - */ -async function scrape({ url }) { - const res = await fetch(url, { - headers, - agent, - }); - const html = await res.text(); - - // Get the metadata from the scraped html. - const metadata = await metascraper({ - html, - url, - }); - return metadata; -} /** * Updates an Asset based on scraped asset metadata. @@ -64,16 +12,9 @@ function update(id, meta) { return Asset.update( { id }, { - $set: { - title: meta.title || '', - description: meta.description || '', - image: meta.image ? meta.image : '', - author: meta.author || '', - publication_date: meta.date || '', - modified_date: meta.modified || '', - section: meta.section || '', + $set: merge(meta, { scraped: new Date(), - }, + }), } ); } @@ -95,7 +36,7 @@ module.exports = () => { } // Scrape the metadata from the asset. - const meta = await scrape(asset); + const meta = await scraper.scrape(asset.url); log.info('Finished scraping'); diff --git a/services/scraper.js b/services/scraper.js deleted file mode 100644 index cfd845424..000000000 --- a/services/scraper.js +++ /dev/null @@ -1,33 +0,0 @@ -const kue = require('./kue'); - -/** - * Exposes a service object to allow operations to execute against the scraper. - * @type {Object} - */ -const scraper = { - /** - * Create the new Task kue singleton. - */ - task: new kue.Task({ - name: 'scraper', - }), - - /** - * Creates a new scraper job and scrapes the url when it gets processed. - */ - async create(ctx, id) { - ctx.log.info({ assetID: id }, 'Creating job'); - - const job = await scraper.task.create({ - title: `Scrape for asset ${id}`, - id: ctx.id, - asset_id: id, - }); - - ctx.log.info({ jobID: job.id, assetID: id }, 'Created job'); - - return job; - }, -}; - -module.exports = scraper; diff --git a/services/scraper/index.js b/services/scraper/index.js new file mode 100644 index 000000000..59faa8bf0 --- /dev/null +++ b/services/scraper/index.js @@ -0,0 +1,96 @@ +const fetch = require('node-fetch'); +const ProxyAgent = require('proxy-agent'); +const { merge } = require('lodash'); + +const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config'); +const kue = require('../kue'); +const { version } = require('../../package.json'); + +// Load the scraper with the rules. +const metascraper = require('metascraper').load([ + require('metascraper-title')(), + require('metascraper-description')(), + require('metascraper-image')(), + require('metascraper-author')(), + require('metascraper-date')(), + require('./rules/modified')(), + require('./rules/section')(), +]); + +let customHeaders = {}; +try { + customHeaders = JSON.parse(SCRAPER_HEADERS); +} catch (err) { + console.error('Cannot parse TALK_SCRAPER_HEADERS'); + throw err; +} + +// Parse the headers to be added to the scraper. +const headers = merge( + { + 'User-Agent': `Coral-Talk/${version}`, + }, + customHeaders +); + +// Add proxy configuration if exists. +const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null; + +/** + * Exposes a service object to allow operations to execute against the scraper. + * @type {Object} + */ +const scraper = { + /** + * Create the new Task kue singleton. + */ + task: new kue.Task({ + name: 'scraper', + }), + + /** + * Creates a new scraper job and scrapes the url when it gets processed. + */ + async create(ctx, id) { + ctx.log.info({ assetID: id }, 'Creating job'); + + const job = await scraper.task.create({ + title: `Scrape for asset ${id}`, + id: ctx.id, + asset_id: id, + }); + + ctx.log.info({ jobID: job.id, assetID: id }, 'Created job'); + + return job; + }, + + /** + * Scrapes the given asset for metadata. + */ + async scrape(url) { + const res = await fetch(url, { + headers, + agent, + }); + const html = await res.text(); + + // Get the metadata from the scraped html. + const meta = await metascraper({ + html, + url, + }); + + return { + title: meta.title || '', + description: meta.description || '', + image: meta.image ? meta.image : '', + author: meta.author || '', + publication_date: meta.date || '', + modified_date: meta.modified || '', + section: meta.section || '', + }; + }, +}; + +module.exports = scraper; diff --git a/jobs/scraper/rules/modified.js b/services/scraper/rules/modified.js similarity index 100% rename from jobs/scraper/rules/modified.js rename to services/scraper/rules/modified.js diff --git a/jobs/scraper/rules/section.js b/services/scraper/rules/section.js similarity index 100% rename from jobs/scraper/rules/section.js rename to services/scraper/rules/section.js