replaced eslint:recommended with prettier

This commit is contained in:
Wyatt Johnson
2018-01-11 20:00:34 -07:00
parent d56c19016a
commit 0abc2ca243
649 changed files with 16235 additions and 13008 deletions
+41 -34
View File
@@ -10,73 +10,75 @@ const metascraper = require('metascraper');
* @type {Object}
*/
const scraper = {
/**
* Create the new Task kue singleton.
*/
task: new kue.Task({
name: 'scraper'
name: 'scraper',
}),
/**
* Creates a new scraper job and scrapes the url when it gets processed.
*/
create(asset) {
debug(`Creating job for Asset[${asset.id}]`);
return scraper.task.create({
title: `Scrape for asset ${asset.id}`,
asset_id: asset.id
}).then((job) => {
return scraper.task
.create({
title: `Scrape for asset ${asset.id}`,
asset_id: asset.id,
})
.then(job => {
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
return job;
});
return job;
});
},
/**
* Scrapes the given asset for metadata.
*/
async scrape(asset) {
return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, {
section: ($) => $('meta[property="article:section"]').attr('content'),
modified: ($) => $('meta[property="article:modified"]').attr('content')
}));
return metascraper.scrapeUrl(
asset.url,
Object.assign({}, metascraper.RULES, {
section: $ => $('meta[property="article:section"]').attr('content'),
modified: $ => $('meta[property="article:modified"]').attr('content'),
})
);
},
/**
* Updates an Asset based on scraped asset metadata.
*/
update(id, meta) {
return AssetModel.update({id}, {
$set: {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
scraped: new Date()
return AssetModel.update(
{ id },
{
$set: {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
scraped: new Date(),
},
}
});
);
},
/**
* Start the queue processor for the scraper job.
*/
process() {
debug(`Now processing ${scraper.task.name} jobs`);
scraper.task.process(async (job, done) => {
debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`);
try {
// Find the asset, or complain that it doesn't exist.
const asset = await AssetsService.findById(job.data.asset_id);
if (!asset) {
@@ -86,21 +88,26 @@ const scraper = {
// Scrape the metadata from the asset.
const meta = await scraper.scrape(asset);
debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`);
debug(
`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${
job.data.asset_id
}]`
);
// Assign the metadata retrieved for the asset to the db.
await scraper.update(job.data.asset_id, meta);
} catch (err) {
debug(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err);
debug(
`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`,
err
);
return done(err);
}
debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`);
done();
});
}
},
};
module.exports = scraper;