#!/usr/bin/env node /** * Module dependencies. */ const util = require('./util'); const program = require('commander'); const parseDuration = require('ms'); const Table = require('cli-table'); const AssetModel = require('../models/asset'); const CommentModel = require('../models/comment'); const AssetsService = require('../services/assets'); const mongoose = require('../services/mongoose'); const scraper = require('../services/scraper'); const Context = require('../graph/context'); const inquirer = require('inquirer'); const { URL } = require('url'); // Register the shutdown criteria. util.onshutdown([() => mongoose.disconnect()]); /** * Lists all the assets registered in the database. */ async function listAssets(opts) { try { let assets = await AssetModel.find({}).sort({ created_at: 1 }); switch (opts.format) { case 'json': { console.log(JSON.stringify(assets, null, 2)); break; } default: { let table = new Table({ head: ['ID', 'Title', 'URL'], }); assets.forEach(asset => { table.push([ asset.id, asset.title ? asset.title : '', asset.url ? asset.url : '', ]); }); console.log(table.toString()); break; } } util.shutdown(); } catch (e) { console.error(e); util.shutdown(1); } } async function refreshAssets(ageString) { try { const query = AssetModel.find({}, { id: 1 }); if (ageString) { // An age was specified, so filter only those assets. const ageMs = parseDuration(ageString); const age = new Date(Date.now() - ageMs); query.merge({ $or: [ { scraped: { $lte: age, }, }, { scraped: null, }, ], }); } // Create a graph context. const ctx = Context.forSystem(); // Load the assets. const cursor = query.cursor(); // Queue all the assets for scraping. const promises = []; let asset = await cursor.next(); while (asset) { promises.push(scraper.create(ctx, asset.id)); asset = await cursor.next(); } await Promise.all(promises); console.log(`${promises.length} Assets were queued to be scraped.`); util.shutdown(); } catch (e) { console.error(e); util.shutdown(1); } } async function updateURL(assetID, assetURL) { try { await AssetsService.updateURL(assetID, assetURL); console.log(`Asset ${assetID} was updated to have url ${assetURL}.`); util.shutdown(); } catch (e) { console.error(e); util.shutdown(1); } } async function merge(srcID, dstID) { try { // Grab the assets... let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcID, dstID]); if (!srcAsset || !dstAsset) { throw new Error('Not all assets indicated by id exist, cannot merge'); } // Count the affected resources... let srcCommentCount = await CommentModel.find({ asset_id: srcID }).count(); console.log( `Now going to update ${srcCommentCount} comments and delete the source Asset[${srcID}].` ); let { confirm } = await inquirer.prompt([ { type: 'confirm', name: 'confirm', message: 'Proceed with merge', default: false, }, ]); if (confirm) { // Perform the merge! await AssetsService.merge(srcID, dstID); } else { console.warn('Aborting merge'); } util.shutdown(0); } catch (e) { console.error(e); util.shutdown(1); } } async function rewrite(search, replace, options) { try { search = new RegExp(search); const assets = await AssetModel.find({ url: { $regex: search }, }); if (assets.length === 0) { console.log(`No assets found with the pattern: ${search}`); return util.shutdown(0); } let opts = []; assets.forEach(({ id, url: oldURL }) => { // Replace the url. const newURL = oldURL.replace(search, replace); // Try to validate that the new url is valid. try { new URL(newURL); } catch (err) { throw new Error( `Rewrite would have replaced the valid URL ${oldURL} with an invalid one ${newURL}` ); } opts.push({ find: { id }, updateOne: { $set: { url: newURL } }, id, oldURL, newURL, }); }); if (opts.length > 0) { if (options.dryRun) { const table = new Table({ head: ['ID', 'Old URL', 'New URL'] }); opts.forEach(({ id, oldURL, newURL }) => { table.push([id, oldURL, newURL]); }); console.log(table.toString()); } else { const bulk = AssetModel.collection.initializeUnorderedBulkOp(); opts.forEach(({ find, updateOne, oldURL, newURL }) => { // If the url was updated with the operation, then queue up the update op. if (newURL !== oldURL) { bulk.find(find).updateOne(updateOne); } }); await bulk.execute(); console.log(`${opts.length} assets had their url's updated`); } } util.shutdown(0); } catch (err) { console.error(err); util.shutdown(1); } } //============================================================================== // Setting up the program command line arguments. //============================================================================== program .command('list') .option( '--format ', 'Specify the output format [table]', /^(table|json)$/i, 'table' ) .description('list all the assets in the database') .action(listAssets); program .command('refresh [age]') .description('queues the assets that exceed the age requested') .action(refreshAssets); program .command('update-url ') .description('update the URL of an asset') .action(updateURL); program .command('merge ') .description( 'merges two assets together by moving comments from src to dst and deleting the src asset' ) .action(merge); program .command('rewrite ') .option('-d, --dry-run', 'enables dry run of the replacement') .description( "rewrites asset url's using the provided regex replacement pattern" ) .action(rewrite); program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); util.shutdown(); }