wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-07-11 22:52:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Metadata Scraper Debugger (#2048)

Browse Source 
* feat: added asset scrape debugger

* fix: added docs for scraper

* fix: added asset scraping to sidebar
This commit is contained in:
Wyatt Johnson
2018-10-30 17:02:33 +00:00
committed by GitHub
parent 399f5ef7ba
commit 7d246bba76
8 changed files with 175 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/cli-assets
+29
View File
@@ -219,6 +219,28 @@ async function rewrite(search, replace, options) {
}
}
/**
* debugAssetURL will scrape the given URL and print it's scraped values.
*/
async function debugAssetURL(url) {
try {
const meta = await scraper.scrape(url);
const table = new Table({ head: ['Property', 'Value'] });
for (const [property, value] of Object.entries(meta)) {
table.push([property, value]);
}
console.log(table.toString());
util.shutdown(0);
} catch (err) {
console.error(err);
util.shutdown(1);
}
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
@@ -234,6 +256,13 @@ program
.description('list all the assets in the database')
.action(listAssets);
program
.command('debug [url]')
.description(
'prints the scraped metadata that would be added to the given asset'
)
.action(debugAssetURL);
program
.command('refresh [age]')
.description('queues the assets that exceed the age requested')
docs/_config.yml
+2
View File
@@ -104,6 +104,8 @@ sidebar:
url: /integrating/authentication/
- title: Asset Management
url: /integrating/asset-management/
- title: Asset Scraping
url: /integrating/asset-scraping/
- title: Configuring the Comment Stream
url: /integrating/configuring-comment-stream/
- title: Configuring the Admin
docs/source/integrating/asset-scraping.md
+44
View File
@@ -0,0 +1,44 @@
---
title: Asset Scraping
permalink: /integrating/asset-scraping/
---
By default, Assets in Talk have their metadata scraped when they are loaded.
This provides the easiest way for newsrooms to integrate their CMS's into Talk
in a simple way. We use the following
[meta tags](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta) on
the target pages that allow us to extract some properties.
| Asset Property | Selector |
|--------------------|----------|
| `title` | See [`metascraper-title`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-title/index.js) |
| `description` | See [`metascraper-description`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-description/index.js) |
| `image` | See [`metascraper-image`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-image/index.js) |
| `author` | See [`metascraper-author`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-author/index.js) |
| `publication_date` | See [`metascraper-date`](https://github.com/microlinkhq/metascraper/blob/dc664c37ea1b238b1e3e9d5342edfacc9027892c/packages/metascraper-date/index.js) |
| `modified_date` | `meta[property="article:modified"]` |
| `section` | `meta[property="article:section"]` |
You can use the `./bin/cli assets debug <url>` to print the scraped metadata
from that URL. For example:
```bash
$ ./bin/cli assets debug https://www.washingtonpost.com/technology/2018/10/30/apple-event-october-ipad-pro-macbook-air/
┌──────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Property │ Value │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ title │ Apple redesigns the iPad Pro, breathes new life in the MacBook Air │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ description │ Apple is unveiling new iPads and MacBooks at an event in New York starting at 10 a.m. Fowler is there and will report in with the news and hands-on analysis throughout the day. │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ image │ https://www.washingtonpost.com/resizer/JAwNQE2alL2JjiWrbXeJ46wZHqA=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/G5TWBFW4LAI6RC5MX7QB7TODUY.jpg │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ author │ Geoffrey A. Fowler │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ publication_date │ 2018-10-30T10:40:00.000Z │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ modified_date │ │
├──────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ section │ │
└──────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
jobs/scraper/index.js
+4 -63
View File
@@ -1,61 +1,9 @@
const Asset = require('../../models/asset');
const scraper = require('../../services/scraper');
const scraper = require('../../services/scraper/scraper');
const Assets = require('../../services/assets');
const { createLogger } = require('../../services/logging');
const logger = createLogger('jobs:scraper');
const fetch = require('node-fetch');
const { merge } = require('lodash');
const { version } = require('../../package.json');
const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config');
const ProxyAgent = require('proxy-agent');
// Load the scraper with the rules.
const metascraper = require('metascraper').load([
require('metascraper-title')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-author')(),
require('metascraper-date')(),
require('./rules/modified')(),
require('./rules/section')(),
]);
let customHeaders = {};
try {
customHeaders = JSON.parse(SCRAPER_HEADERS);
} catch (err) {
console.error('Cannot parse TALK_SCRAPER_HEADERS');
throw err;
}
// Parse the headers to be added to the scraper.
const headers = merge(
{
'User-Agent': `Coral-Talk/${version}`,
},
customHeaders
);
// Add proxy configuration if exists.
const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null;
/**
* Scrapes the given asset for metadata.
*/
async function scrape({ url }) {
const res = await fetch(url, {
headers,
agent,
});
const html = await res.text();
// Get the metadata from the scraped html.
const metadata = await metascraper({
html,
url,
});
return metadata;
}
/**
* Updates an Asset based on scraped asset metadata.
@@ -64,16 +12,9 @@ function update(id, meta) {
return Asset.update(
{ id },
{
$set: {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
$set: merge(meta, {
scraped: new Date(),
},
}),
}
);
}
@@ -95,7 +36,7 @@ module.exports = () => {
}
// Scrape the metadata from the asset.
const meta = await scrape(asset);
const meta = await scraper.scrape(asset.url);
log.info('Finished scraping');
services/scraper.js
-33
View File
@@ -1,33 +0,0 @@
const kue = require('./kue');
/**
* Exposes a service object to allow operations to execute against the scraper.
* @type {Object}
*/
const scraper = {
/**
* Create the new Task kue singleton.
*/
task: new kue.Task({
name: 'scraper',
}),
/**
* Creates a new scraper job and scrapes the url when it gets processed.
*/
async create(ctx, id) {
ctx.log.info({ assetID: id }, 'Creating job');
const job = await scraper.task.create({
title: `Scrape for asset ${id}`,
id: ctx.id,
asset_id: id,
});
ctx.log.info({ jobID: job.id, assetID: id }, 'Created job');
return job;
},
};
module.exports = scraper;
services/scraper/index.js
+96
View File
@@ -0,0 +1,96 @@
const fetch = require('node-fetch');
const ProxyAgent = require('proxy-agent');
const { merge } = require('lodash');
const { SCRAPER_HEADERS, SCRAPER_PROXY_URL } = require('../../config');
const kue = require('../kue');
const { version } = require('../../package.json');
// Load the scraper with the rules.
const metascraper = require('metascraper').load([
require('metascraper-title')(),
require('metascraper-description')(),
require('metascraper-image')(),
require('metascraper-author')(),
require('metascraper-date')(),
require('./rules/modified')(),
require('./rules/section')(),
]);
let customHeaders = {};
try {
customHeaders = JSON.parse(SCRAPER_HEADERS);
} catch (err) {
console.error('Cannot parse TALK_SCRAPER_HEADERS');
throw err;
}
// Parse the headers to be added to the scraper.
const headers = merge(
{
'User-Agent': `Coral-Talk/${version}`,
},
customHeaders
);
// Add proxy configuration if exists.
const agent = SCRAPER_PROXY_URL ? new ProxyAgent(SCRAPER_PROXY_URL) : null;
/**
* Exposes a service object to allow operations to execute against the scraper.
* @type {Object}
*/
const scraper = {
/**
* Create the new Task kue singleton.
*/
task: new kue.Task({
name: 'scraper',
}),
/**
* Creates a new scraper job and scrapes the url when it gets processed.
*/
async create(ctx, id) {
ctx.log.info({ assetID: id }, 'Creating job');
const job = await scraper.task.create({
title: `Scrape for asset ${id}`,
id: ctx.id,
asset_id: id,
});
ctx.log.info({ jobID: job.id, assetID: id }, 'Created job');
return job;
},
/**
* Scrapes the given asset for metadata.
*/
async scrape(url) {
const res = await fetch(url, {
headers,
agent,
});
const html = await res.text();
// Get the metadata from the scraped html.
const meta = await metascraper({
html,
url,
});
return {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
};
},
};
module.exports = scraper;
jobs/scraper/rules/modified.js → services/scraper/rules/modified.js
View File
jobs/scraper/rules/section.js → services/scraper/rules/section.js
View File