talk/bin/cli-assets

#!/usr/bin/env node

/**
 * Module dependencies.
 */

const util = require('./util');
const program = require('commander');
const parseDuration = require('ms');
const Table = require('cli-table2');
const AssetModel = require('../models/asset');
const CommentModel = require('../models/comment');
const AssetsService = require('../services/assets');
const mongoose = require('../services/mongoose');
const scraper = require('../services/scraper');
const Context = require('../graph/context');
const inquirer = require('inquirer');
const { URL } = require('url');

// Register the shutdown criteria.
util.onshutdown([() => mongoose.disconnect()]);

/**
 * Lists all the assets registered in the database.
 */
async function listAssets(opts) {
  try {
    let assets = await AssetModel.find({}).sort({ created_at: 1 });

    switch (opts.format) {
      case 'json': {
        console.log(JSON.stringify(assets, null, 2));
        break;
      }
      default: {
        let table = new Table({
          head: ['ID', 'Title', 'URL'],
        });

        assets.forEach(asset => {
          table.push([
            asset.id,
            asset.title ? asset.title : '',
            asset.url ? asset.url : '',
          ]);
        });

        console.log(table.toString());
        break;
      }
    }

    util.shutdown();
  } catch (e) {
    console.error(e);
    util.shutdown(1);
  }
}

async function refreshAssets(ageString) {
  try {
    const query = AssetModel.find({}, { id: 1 });
    if (ageString) {
      // An age was specified, so filter only those assets.
      const ageMs = parseDuration(ageString);
      const age = new Date(Date.now() - ageMs);

      query.merge({
        $or: [
          {
            scraped: {
              $lte: age,
            },
          },
          {
            scraped: null,
          },
        ],
      });
    }

    // Create a graph context.
    const ctx = Context.forSystem();

    // Load the assets.
    const cursor = query.cursor();

    // Queue all the assets for scraping.
    const promises = [];

    let asset = await cursor.next();
    while (asset) {
      promises.push(scraper.create(ctx, asset.id));
      asset = await cursor.next();
    }

    await Promise.all(promises);

    console.log(`${promises.length} Assets were queued to be scraped.`);

    util.shutdown();
  } catch (e) {
    console.error(e);
    util.shutdown(1);
  }
}

async function updateURL(assetID, assetURL) {
  try {
    await AssetsService.updateURL(assetID, assetURL);

    console.log(`Asset ${assetID} was updated to have url ${assetURL}.`);
    util.shutdown();
  } catch (e) {
    console.error(e);
    util.shutdown(1);
  }
}

async function merge(srcID, dstID) {
  try {
    // Grab the assets...
    let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcID, dstID]);
    if (!srcAsset || !dstAsset) {
      throw new Error('Not all assets indicated by id exist, cannot merge');
    }

    // Count the affected resources...
    let srcCommentCount = await CommentModel.find({ asset_id: srcID }).count();

    console.log(
      `Now going to update ${srcCommentCount} comments and delete the source Asset[${srcID}].`
    );

    let { confirm } = await inquirer.prompt([
      {
        type: 'confirm',
        name: 'confirm',
        message: 'Proceed with merge',
        default: false,
      },
    ]);

    if (confirm) {
      // Perform the merge!
      await AssetsService.merge(srcID, dstID);
    } else {
      console.warn('Aborting merge');
    }

    util.shutdown(0);
  } catch (e) {
    console.error(e);
    util.shutdown(1);
  }
}

async function rewrite(search, replace, options) {
  try {
    search = new RegExp(search);

    const assets = await AssetModel.find({
      url: { $regex: search },
    });
    if (assets.length === 0) {
      console.log(`No assets found with the pattern: ${search}`);
      return util.shutdown(0);
    }

    let opts = [];
    assets.forEach(({ id, url: oldURL }) => {
      // Replace the url.
      const newURL = oldURL.replace(search, replace);

      // Try to validate that the new url is valid.
      try {
        new URL(newURL);
      } catch (err) {
        throw new Error(
          `Rewrite would have replaced the valid URL ${oldURL} with an invalid one ${newURL}`
        );
      }

      opts.push({
        find: { id },
        updateOne: { $set: { url: newURL } },
        id,
        oldURL,
        newURL,
      });
    });

    if (opts.length > 0) {
      if (options.dryRun) {
        const table = new Table({ head: ['ID', 'Old URL', 'New URL'] });

        opts.forEach(({ id, oldURL, newURL }) => {
          table.push([id, oldURL, newURL]);
        });

        console.log(table.toString());
      } else {
        const bulk = AssetModel.collection.initializeUnorderedBulkOp();
        opts.forEach(({ find, updateOne, oldURL, newURL }) => {
          // If the url was updated with the operation, then queue up the update op.
          if (newURL !== oldURL) {
            bulk.find(find).updateOne(updateOne);
          }
        });
        await bulk.execute();
        console.log(`${opts.length} assets had their url's updated`);
      }
    }

    util.shutdown(0);
  } catch (err) {
    console.error(err);
    util.shutdown(1);
  }
}

//==============================================================================
// Setting up the program command line arguments.
//==============================================================================

program
  .command('list')
  .option(
    '--format <type>',
    'Specify the output format [table]',
    /^(table|json)$/i,
    'table'
  )
  .description('list all the assets in the database')
  .action(listAssets);

program
  .command('refresh [age]')
  .description('queues the assets that exceed the age requested')
  .action(refreshAssets);

program
  .command('update-url <assetID> <url>')
  .description('update the URL of an asset')
  .action(updateURL);

program
  .command('merge <srcID> <dstID>')
  .description(
    'merges two assets together by moving comments from src to dst and deleting the src asset'
  )
  .action(merge);

program
  .command('rewrite <search> <replace>')
  .option('-d, --dry-run', 'enables dry run of the replacement')
  .description(
    "rewrites asset url's using the provided regex replacement pattern"
  )
  .action(rewrite);

program.parse(process.argv);

// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
  program.outputHelp();
  util.shutdown();
}