From ca3d56def87d1e478fb44a3d29ce82d1dd47ad1d Mon Sep 17 00:00:00 2001
From: Wyatt Johnson <wyattjoh@gmail.com>
Date: Wed, 9 May 2018 14:40:24 -0600
Subject: [PATCH] asset refresh patched

---
 bin/cli-assets | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/bin/cli-assets b/bin/cli-assets
index 62c996412..5c1afb741 100755
--- a/bin/cli-assets
+++ b/bin/cli-assets
@@ -59,12 +59,13 @@ async function listAssets(opts) {
 
 async function refreshAssets(ageString) {
   try {
-    const now = new Date().getTime();
-    const ageMs = parseDuration(ageString);
-    const age = new Date(now - ageMs);
+    const query = AssetModel.find({}, { id: 1 });
+    if (ageString) {
+      // An age was specified, so filter only those assets.
+      const ageMs = parseDuration(ageString);
+      const age = new Date(Date.now() - ageMs);
 
-    let assets = await AssetModel.find(
-      {
+      query.merge({
         $or: [
           {
             scraped: {
@@ -75,16 +76,28 @@ async function refreshAssets(ageString) {
             scraped: null,
           },
         ],
-      },
-      { id: 1 }
-    );
+      });
+    }
 
     // Create a graph context.
     const ctx = Context.forSystem();
 
+    // Load the assets.
+    const cursor = query.cursor();
+
     // Queue all the assets for scraping.
-    await Promise.all(assets.map(({ id }) => scraper.create(ctx, id)));
-    console.log('Assets were queued to be scraped');
+    const promises = [];
+
+    let asset = await cursor.next();
+    while (asset) {
+      promises.push(scraper.create(ctx, asset.id));
+      asset = await cursor.next();
+    }
+
+    await Promise.all(promises);
+
+    console.log(`${promises.length} Assets were queued to be scraped.`);
+
     util.shutdown();
   } catch (e) {
     console.error(e);
@@ -222,7 +235,7 @@ program
   .action(listAssets);
 
 program
-  .command('refresh <age>')
+  .command('refresh [age]')
   .description('queues the assets that exceed the age requested')
   .action(refreshAssets);