added asset merging + url updates

bin/cli-assets

@@ -8,9 +8,12 @@ const program = require('./commander');
 const parseDuration = require('ms');
 const Table = require('cli-table');
 const AssetModel = require('../models/asset');
+const CommentModel = require('../models/comment');
+const AssetsService = require('../services/assets');
 const mongoose = require('../services/mongoose');
 const scraper = require('../services/scraper');
 const util = require('./util');
+const inquirer = require('inquirer');
 
 // Register the shutdown criteria.
 util.onshutdown([
@@ -20,65 +23,112 @@ util.onshutdown([
 /**
  * Lists all the assets registered in the database.
  */
-function listAssets() {
-  AssetModel
-    .find({})
-    .sort({'created_at': 1})
-    .then((asset) => {
-      let table = new Table({
-        head: [
-          'ID',
-          'Title',
-          'URL'
-        ]
-      });
+async function listAssets() {
+  try {
+    let assets = await AssetModel.find({}).sort({'created_at': 1});
 
-      asset.forEach((asset) => {
-        table.push([
-          asset.id,
-          asset.title ? asset.title : '',
-          asset.url ? asset.url : ''
-        ]);
-      });
-
-      console.log(table.toString());
-      util.shutdown();
-    })
-    .catch((err) => {
-      console.error(err);
-      util.shutdown(1);
+    let table = new Table({
+      head: [
+        'ID',
+        'Title',
+        'URL'
+      ]
     });
+
+    assets.forEach((asset) => {
+      table.push([
+        asset.id,
+        asset.title ? asset.title : '',
+        asset.url ? asset.url : ''
+      ]);
+    });
+
+    console.log(table.toString());
+    util.shutdown();
+  } catch (e) {
+    console.error(e);
+    util.shutdown(1);
+  }
 }
 
-function refreshAssets(ageString) {
-  const now = new Date().getTime();
-  const ageMs = parseDuration(ageString);
-  const age = new Date(now - ageMs);
+async function refreshAssets(ageString) {
+  try {
+    const now = new Date().getTime();
+    const ageMs = parseDuration(ageString);
+    const age = new Date(now - ageMs);
 
-  AssetModel.find({
-    $or: [
-      {
-        scraped: {
-          $lte: age
+    let assets = await AssetModel.find({
+      $or: [
+        {
+          scraped: {
+            $lte: age
+          }
+        },
+        {
+          scraped: null
         }
-      },
-      {
-        scraped: null
-      }
-    ]
-  })
+      ]
+    });
 
-  // Queue all the assets for scraping.
-  .then((assets) => Promise.all(assets.map(scraper.create)))
+    // Queue all the assets for scraping.
+    await Promise.all(assets.map(scraper.create));
 
-  .then(() => {
     console.log('Assets were queued to be scraped');
     util.shutdown();
-  })
-  .catch((err) => {
-    console.error(err);
+  } catch (e) {
+    console.error(e);
     util.shutdown(1);
-  });
+  }
+}
+
+async function updateURL(assetID, assetURL) {
+  try {
+    await AssetsService.updateURL(assetID, assetURL);
+
+    console.log(`Asset ${assetID} was updated to have url ${assetURL}.`);
+    util.shutdown();
+  } catch (e) {
+    console.error(e);
+    util.shutdown(1);
+  }
+}
+
+async function merge(srcID, dstID) {
+  try {
+
+    // Grab the assets...
+    let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcID, dstID]);
+    if (!srcAsset || !dstAsset) {
+      throw new Error('Not all assets indicated by id exist, cannot merge');
+    }
+
+    // Count the affected resources...
+    let srcCommentCount = await CommentModel.find({asset_id: srcID}).count();
+
+    console.log(`Now going to update ${srcCommentCount} comments and delete the source Asset[${srcID}].`);
+
+    let {confirm} = await inquirer.prompt([
+      {
+        type: 'confirm',
+        name: 'confirm',
+        message: 'Proceed with merge',
+        default: false
+      }
+    ]);
+
+    if (confirm) {
+
+      // Perform the merge!
+      await AssetsService.merge(srcID, dstID);
+    } else {
+      console.warn('Aborting merge');
+    }
+
+    util.shutdown(0);
+  } catch (e) {
+    console.error(e);
+    util.shutdown(1);
+  }
 }
 
 //==============================================================================
@@ -95,6 +145,16 @@ program
   .description('queues the assets that exceed the age requested')
   .action(refreshAssets);
 
+program
+  .command('update-url <assetID> <url>')
+  .description('update the URL of an asset')
+  .action(updateURL);
+
+program
+  .command('merge <srcID> <dstID>')
+  .description('merges two assets together by moving comments from src to dst and deleting the src asset')
+  .action(merge);
+
 program.parse(process.argv);
 
 // If there is no command listed, output help.

errors.js

@@ -168,6 +168,13 @@ const ErrCommentTooShort = new APIError('Comment was too short', {
   status: 400
 });
 
+// ErrAssetURLAlreadyExists is returned when a rename operation is requested
+// but an asset already exists with the new url.
+const ErrAssetURLAlreadyExists = new APIError('Asset URL already exists, cannot rename', {
+  translation_key: 'ASSET_URL_ALREADY_EXISTS',
+  status: 409
+});
+
 module.exports = {
   ExtendableError,
   APIError,
@@ -191,5 +198,6 @@ module.exports = {
   ErrInstallLock,
   ErrLoginAttemptMaximumExceeded,
   ErrEditWindowHasEnded,
-  ErrCommentTooShort
+  ErrCommentTooShort,
+  ErrAssetURLAlreadyExists
 };

services/assets.js

@@ -1,3 +1,4 @@
+const CommentModel = require('../models/comment');
 const AssetModel = require('../models/asset');
 const SettingsService = require('./settings');
 const domainlist = require('./domainlist');
@@ -128,9 +129,61 @@ module.exports = class AssetsService {
    * @param  {Array} ids an array of Strings of asset_id
    * @return {Promise}     resolves to list of Assets
    */
-  static findMultipleById(ids) {
-    const query = ids.map((id) => ({id}));
-    return AssetModel.find(query);
+  static async findByIDs(ids) {
+
+    // Find the assets.
+    let assets = await AssetModel.find({
+      id: {
+        $in: ids
+      }
+    });
+
+    // Return them in the right order.
+    return ids.map((id) => assets.find((asset) => asset.id === id));
+  }
+
+  static async updateURL(id, url) {
+
+    // Try to see if an asset already exists with the given url.
+    let asset = await AssetsService.findByUrl(url);
+    if (asset !== null) {
+      throw errors.ErrAssetURLAlreadyExists;
+    }
+
+    // Seems that there was no other asset with the same url, try and perform
+    // the rename operation! An error may be thrown from this if the operation
+    // fails. This is ok.
+    await AssetModel.update({id}, {$set: {url}});
+  }
+
+  static async merge(srcAssetID, dstAssetID) {
+
+    // Fetch both assets.
+    let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcAssetID, dstAssetID]);
+    if (!srcAsset || !dstAsset) {
+      throw errors.ErrNotFound;
+    }
+
+    // Resolve the merge operation, this invloves moving all resources attached
+    // to the src asset to the dst asset, and then removing the src asset.
+
+    // First, update all the old comments to the new asset.
+    await CommentModel.update({
+      asset_id: srcAssetID
+    }, {
+      $set: {
+        asset_id: dstAssetID
+      }
+    }, {
+      multi: true
+    });
+
+    // Second remove the old asset.
+    await AssetModel.remove({
+      id: srcAssetID
+    });
+
+    // That's it!
   }
 
   static all(skip = null, limit = null) {

test/server/services/assets.js

@@ -1,22 +1,29 @@
 const AssetModel = require('../../../models/asset');
+const CommentModel = require('../../../models/comment');
 const AssetsService = require('../../../services/assets');
+const CommentsService = require('../../../services/comments');
 const SettingsService = require('../../../services/settings');
 
+const url = require('url');
 const chai = require('chai');
 const expect = chai.expect;
+const chaiAsPromised = require('chai-as-promised');
+
+chai.use(chaiAsPromised);
 
 // Use the chai should.
 chai.should();
 
+const settings = {id: '1', moderation: 'PRE', domains: {whitelist: ['new.test.com', 'test.com', 'override.test.com']}};
+const defaults = {url:'http://test.com'};
+
 describe('services.AssetsService', () => {
 
-  beforeEach(() => {
-    const settings = {id: '1', moderation: 'PRE', domains: {whitelist: ['new.test.com', 'test.com', 'override.test.com']}};
-    const defaults = {url:'http://test.com'};
+  let asset;
+  beforeEach(async () => {
+    await SettingsService.init(settings);
 
-    return SettingsService.init(settings).then(() => {
-      return AssetModel.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true});
-    });
+    asset = await AssetModel.findOneAndUpdate({id: '1'}, {$setOnInsert: defaults}, {upsert: true, new: true});
   });
 
   describe('#findById', ()=> {
@@ -120,4 +127,79 @@ describe('services.AssetsService', () => {
         });
     });
   });
+
+  describe('#updateURL', () => {
+
+    it('should change the url if the asset was found, and there was no conflict', async () => {
+      let newURL = url.resolve(asset.url, '/new-url');
+
+      // Update the asset.
+      await AssetsService.updateURL(asset.id, newURL);
+
+      // Check that the url was updated.
+      let {url: databaseURL} = await AssetsService.findById(asset.id);
+
+      expect(databaseURL).to.equal(newURL);
+    });
+
+    it('should error if the new url already exists', async () => {
+      let newURL = url.resolve(asset.url, '/new-url');
+
+      // Create a new asset with our new URL.
+      await AssetModel.findOneAndUpdate({id: '2'}, {$setOnInsert: {url: newURL}}, {upsert: true, new: true});
+
+      return AssetsService.updateURL(asset.id, newURL).should.eventually.be.rejected;
+    });
+
+  });
+
+  describe('#merge', () => {
+
+    it('should error if either the src or the dst is missing', () => {
+      return AssetsService.merge('not-found', asset.id).should.eventually.be.rejected;
+    });
+
+    it('should merge the assets', async () => {
+      let newURL = url.resolve(asset.url, '/new-url');
+
+      // Create a new asset with our new URL.
+      await AssetModel.findOneAndUpdate({id: '2'}, {$setOnInsert: {url: newURL}}, {upsert: true, new: true});
+
+      // Create some comments on both assets.
+      await CommentsService.publicCreate([
+        {
+          asset_id: '1',
+          body: 'This is a comment!',
+          status: 'ACCEPTED'
+        },
+        {
+          asset_id: '1',
+          body: 'This is a comment!',
+          status: 'ACCEPTED'
+        },
+        {
+          asset_id: '2',
+          body: 'This is a comment!',
+          status: 'ACCEPTED'
+        },
+        {
+          asset_id: '2',
+          body: 'This is a comment!',
+          status: 'ACCEPTED'
+        }
+      ]);
+
+      // Merge all the comments from asset 1 into asset 2, followed by deleting
+      // asset 1.
+      await AssetsService.merge('1', '2');
+
+      // Check to see if the comments are moved.
+      expect(await CommentModel.find({asset_id: '1'}).count()).to.equal(0);
+      expect(await CommentModel.find({asset_id: '2'}).count()).to.equal(4);
+
+      // Check to see if the asset was removed.
+      expect(await AssetModel.findOne({id: '1'})).to.equal(null);
+    });
+
+  });
 });