Merge pull request #1510 from coralproject/asset-scrape-fix

Asset Scraping CLI Fix
This commit is contained in:
Wyatt Johnson
2018-04-09 09:10:58 -06:00
committed by GitHub
+19 -13
View File
@@ -13,6 +13,7 @@ const CommentModel = require('../models/comment');
const AssetsService = require('../services/assets');
const mongoose = require('../services/mongoose');
const scraper = require('../services/scraper');
const Context = require('../graph/context');
const inquirer = require('inquirer');
const { URL } = require('url');
@@ -52,22 +53,27 @@ async function refreshAssets(ageString) {
const ageMs = parseDuration(ageString);
const age = new Date(now - ageMs);
let assets = await AssetModel.find({
$or: [
{
scraped: {
$lte: age,
let assets = await AssetModel.find(
{
$or: [
{
scraped: {
$lte: age,
},
},
},
{
scraped: null,
},
],
});
{
scraped: null,
},
],
},
{ id: 1 }
);
// Create a graph context.
const ctx = Context.forSystem();
// Queue all the assets for scraping.
await Promise.all(assets.map(scraper.create));
await Promise.all(assets.map(({ id }) => scraper.create(ctx, id)));
console.log('Assets were queued to be scraped');
util.shutdown();
} catch (e) {