mirror of
https://github.com/wassname/talk.git
synced 2026-09-12 13:01:11 +08:00
replaced eslint:recommended with prettier
This commit is contained in:
+41
-34
@@ -10,73 +10,75 @@ const metascraper = require('metascraper');
|
||||
* @type {Object}
|
||||
*/
|
||||
const scraper = {
|
||||
|
||||
/**
|
||||
* Create the new Task kue singleton.
|
||||
*/
|
||||
task: new kue.Task({
|
||||
name: 'scraper'
|
||||
name: 'scraper',
|
||||
}),
|
||||
|
||||
/**
|
||||
* Creates a new scraper job and scrapes the url when it gets processed.
|
||||
*/
|
||||
create(asset) {
|
||||
|
||||
debug(`Creating job for Asset[${asset.id}]`);
|
||||
|
||||
return scraper.task.create({
|
||||
title: `Scrape for asset ${asset.id}`,
|
||||
asset_id: asset.id
|
||||
}).then((job) => {
|
||||
return scraper.task
|
||||
.create({
|
||||
title: `Scrape for asset ${asset.id}`,
|
||||
asset_id: asset.id,
|
||||
})
|
||||
.then(job => {
|
||||
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
|
||||
|
||||
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
|
||||
|
||||
return job;
|
||||
});
|
||||
return job;
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Scrapes the given asset for metadata.
|
||||
*/
|
||||
async scrape(asset) {
|
||||
return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, {
|
||||
section: ($) => $('meta[property="article:section"]').attr('content'),
|
||||
modified: ($) => $('meta[property="article:modified"]').attr('content')
|
||||
}));
|
||||
return metascraper.scrapeUrl(
|
||||
asset.url,
|
||||
Object.assign({}, metascraper.RULES, {
|
||||
section: $ => $('meta[property="article:section"]').attr('content'),
|
||||
modified: $ => $('meta[property="article:modified"]').attr('content'),
|
||||
})
|
||||
);
|
||||
},
|
||||
|
||||
/**
|
||||
* Updates an Asset based on scraped asset metadata.
|
||||
*/
|
||||
update(id, meta) {
|
||||
return AssetModel.update({id}, {
|
||||
$set: {
|
||||
title: meta.title || '',
|
||||
description: meta.description || '',
|
||||
image: meta.image ? meta.image : '',
|
||||
author: meta.author || '',
|
||||
publication_date: meta.date || '',
|
||||
modified_date: meta.modified || '',
|
||||
section: meta.section || '',
|
||||
scraped: new Date()
|
||||
return AssetModel.update(
|
||||
{ id },
|
||||
{
|
||||
$set: {
|
||||
title: meta.title || '',
|
||||
description: meta.description || '',
|
||||
image: meta.image ? meta.image : '',
|
||||
author: meta.author || '',
|
||||
publication_date: meta.date || '',
|
||||
modified_date: meta.modified || '',
|
||||
section: meta.section || '',
|
||||
scraped: new Date(),
|
||||
},
|
||||
}
|
||||
});
|
||||
);
|
||||
},
|
||||
|
||||
/**
|
||||
* Start the queue processor for the scraper job.
|
||||
*/
|
||||
process() {
|
||||
|
||||
debug(`Now processing ${scraper.task.name} jobs`);
|
||||
|
||||
scraper.task.process(async (job, done) => {
|
||||
|
||||
debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`);
|
||||
|
||||
try {
|
||||
|
||||
// Find the asset, or complain that it doesn't exist.
|
||||
const asset = await AssetsService.findById(job.data.asset_id);
|
||||
if (!asset) {
|
||||
@@ -86,21 +88,26 @@ const scraper = {
|
||||
// Scrape the metadata from the asset.
|
||||
const meta = await scraper.scrape(asset);
|
||||
|
||||
debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`);
|
||||
debug(
|
||||
`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${
|
||||
job.data.asset_id
|
||||
}]`
|
||||
);
|
||||
|
||||
// Assign the metadata retrieved for the asset to the db.
|
||||
await scraper.update(job.data.asset_id, meta);
|
||||
} catch (err) {
|
||||
|
||||
debug(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err);
|
||||
debug(
|
||||
`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`,
|
||||
err
|
||||
);
|
||||
return done(err);
|
||||
}
|
||||
|
||||
debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`);
|
||||
done();
|
||||
});
|
||||
}
|
||||
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = scraper;
|
||||
|
||||
Reference in New Issue
Block a user