mirror of
https://github.com/wassname/talk.git
synced 2026-07-20 12:40:47 +08:00
Merge pull request #1510 from coralproject/asset-scrape-fix
Asset Scraping CLI Fix
This commit is contained in:
+19
-13
@@ -13,6 +13,7 @@ const CommentModel = require('../models/comment');
|
||||
const AssetsService = require('../services/assets');
|
||||
const mongoose = require('../services/mongoose');
|
||||
const scraper = require('../services/scraper');
|
||||
const Context = require('../graph/context');
|
||||
const inquirer = require('inquirer');
|
||||
const { URL } = require('url');
|
||||
|
||||
@@ -52,22 +53,27 @@ async function refreshAssets(ageString) {
|
||||
const ageMs = parseDuration(ageString);
|
||||
const age = new Date(now - ageMs);
|
||||
|
||||
let assets = await AssetModel.find({
|
||||
$or: [
|
||||
{
|
||||
scraped: {
|
||||
$lte: age,
|
||||
let assets = await AssetModel.find(
|
||||
{
|
||||
$or: [
|
||||
{
|
||||
scraped: {
|
||||
$lte: age,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
scraped: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
{
|
||||
scraped: null,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ id: 1 }
|
||||
);
|
||||
|
||||
// Create a graph context.
|
||||
const ctx = Context.forSystem();
|
||||
|
||||
// Queue all the assets for scraping.
|
||||
await Promise.all(assets.map(scraper.create));
|
||||
|
||||
await Promise.all(assets.map(({ id }) => scraper.create(ctx, id)));
|
||||
console.log('Assets were queued to be scraped');
|
||||
util.shutdown();
|
||||
} catch (e) {
|
||||
|
||||
Reference in New Issue
Block a user