From f929059daf38d6b9539b26abd23a94659cd1b283 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Fri, 9 Jun 2017 13:23:16 -0600 Subject: [PATCH] added asset merging + url updates --- bin/cli-assets | 156 +++++++++++++++++++++++---------- errors.js | 10 ++- services/assets.js | 59 ++++++++++++- test/server/services/assets.js | 94 ++++++++++++++++++-- 4 files changed, 261 insertions(+), 58 deletions(-) diff --git a/bin/cli-assets b/bin/cli-assets index 97b5b043c..4805bead0 100755 --- a/bin/cli-assets +++ b/bin/cli-assets @@ -8,9 +8,12 @@ const program = require('./commander'); const parseDuration = require('ms'); const Table = require('cli-table'); const AssetModel = require('../models/asset'); +const CommentModel = require('../models/comment'); +const AssetsService = require('../services/assets'); const mongoose = require('../services/mongoose'); const scraper = require('../services/scraper'); const util = require('./util'); +const inquirer = require('inquirer'); // Register the shutdown criteria. util.onshutdown([ @@ -20,65 +23,112 @@ util.onshutdown([ /** * Lists all the assets registered in the database. */ -function listAssets() { - AssetModel - .find({}) - .sort({'created_at': 1}) - .then((asset) => { - let table = new Table({ - head: [ - 'ID', - 'Title', - 'URL' - ] - }); +async function listAssets() { + try { + let assets = await AssetModel.find({}).sort({'created_at': 1}); - asset.forEach((asset) => { - table.push([ - asset.id, - asset.title ? asset.title : '', - asset.url ? asset.url : '' - ]); - }); - - console.log(table.toString()); - util.shutdown(); - }) - .catch((err) => { - console.error(err); - util.shutdown(1); + let table = new Table({ + head: [ + 'ID', + 'Title', + 'URL' + ] }); + + assets.forEach((asset) => { + table.push([ + asset.id, + asset.title ? asset.title : '', + asset.url ? asset.url : '' + ]); + }); + + console.log(table.toString()); + util.shutdown(); + } catch (e) { + console.error(e); + util.shutdown(1); + } } -function refreshAssets(ageString) { - const now = new Date().getTime(); - const ageMs = parseDuration(ageString); - const age = new Date(now - ageMs); +async function refreshAssets(ageString) { + try { + const now = new Date().getTime(); + const ageMs = parseDuration(ageString); + const age = new Date(now - ageMs); - AssetModel.find({ - $or: [ - { - scraped: { - $lte: age + let assets = await AssetModel.find({ + $or: [ + { + scraped: { + $lte: age + } + }, + { + scraped: null } - }, - { - scraped: null - } - ] - }) + ] + }); - // Queue all the assets for scraping. - .then((assets) => Promise.all(assets.map(scraper.create))) + // Queue all the assets for scraping. + await Promise.all(assets.map(scraper.create)); - .then(() => { console.log('Assets were queued to be scraped'); util.shutdown(); - }) - .catch((err) => { - console.error(err); + } catch (e) { + console.error(e); util.shutdown(1); - }); + } +} + +async function updateURL(assetID, assetURL) { + try { + await AssetsService.updateURL(assetID, assetURL); + + console.log(`Asset ${assetID} was updated to have url ${assetURL}.`); + util.shutdown(); + } catch (e) { + console.error(e); + util.shutdown(1); + } +} + +async function merge(srcID, dstID) { + try { + + // Grab the assets... + let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcID, dstID]); + if (!srcAsset || !dstAsset) { + throw new Error('Not all assets indicated by id exist, cannot merge'); + } + + // Count the affected resources... + let srcCommentCount = await CommentModel.find({asset_id: srcID}).count(); + + console.log(`Now going to update ${srcCommentCount} comments and delete the source Asset[${srcID}].`); + + let {confirm} = await inquirer.prompt([ + { + type: 'confirm', + name: 'confirm', + message: 'Proceed with merge', + default: false + } + ]); + + if (confirm) { + + // Perform the merge! + await AssetsService.merge(srcID, dstID); + } else { + console.warn('Aborting merge'); + } + + util.shutdown(0); + } catch (e) { + console.error(e); + util.shutdown(1); + } } //============================================================================== @@ -95,6 +145,16 @@ program .description('queues the assets that exceed the age requested') .action(refreshAssets); +program + .command('update-url ') + .description('update the URL of an asset') + .action(updateURL); + +program + .command('merge ') + .description('merges two assets together by moving comments from src to dst and deleting the src asset') + .action(merge); + program.parse(process.argv); // If there is no command listed, output help. diff --git a/errors.js b/errors.js index c203b1d25..87c3a2a65 100644 --- a/errors.js +++ b/errors.js @@ -168,6 +168,13 @@ const ErrCommentTooShort = new APIError('Comment was too short', { status: 400 }); +// ErrAssetURLAlreadyExists is returned when a rename operation is requested +// but an asset already exists with the new url. +const ErrAssetURLAlreadyExists = new APIError('Asset URL already exists, cannot rename', { + translation_key: 'ASSET_URL_ALREADY_EXISTS', + status: 409 +}); + module.exports = { ExtendableError, APIError, @@ -191,5 +198,6 @@ module.exports = { ErrInstallLock, ErrLoginAttemptMaximumExceeded, ErrEditWindowHasEnded, - ErrCommentTooShort + ErrCommentTooShort, + ErrAssetURLAlreadyExists }; diff --git a/services/assets.js b/services/assets.js index a86c4281f..b81513b72 100644 --- a/services/assets.js +++ b/services/assets.js @@ -1,3 +1,4 @@ +const CommentModel = require('../models/comment'); const AssetModel = require('../models/asset'); const SettingsService = require('./settings'); const domainlist = require('./domainlist'); @@ -128,9 +129,61 @@ module.exports = class AssetsService { * @param {Array} ids an array of Strings of asset_id * @return {Promise} resolves to list of Assets */ - static findMultipleById(ids) { - const query = ids.map((id) => ({id})); - return AssetModel.find(query); + static async findByIDs(ids) { + + // Find the assets. + let assets = await AssetModel.find({ + id: { + $in: ids + } + }); + + // Return them in the right order. + return ids.map((id) => assets.find((asset) => asset.id === id)); + } + + static async updateURL(id, url) { + + // Try to see if an asset already exists with the given url. + let asset = await AssetsService.findByUrl(url); + if (asset !== null) { + throw errors.ErrAssetURLAlreadyExists; + } + + // Seems that there was no other asset with the same url, try and perform + // the rename operation! An error may be thrown from this if the operation + // fails. This is ok. + await AssetModel.update({id}, {$set: {url}}); + } + + static async merge(srcAssetID, dstAssetID) { + + // Fetch both assets. + let [srcAsset, dstAsset] = await AssetsService.findByIDs([srcAssetID, dstAssetID]); + if (!srcAsset || !dstAsset) { + throw errors.ErrNotFound; + } + + // Resolve the merge operation, this invloves moving all resources attached + // to the src asset to the dst asset, and then removing the src asset. + + // First, update all the old comments to the new asset. + await CommentModel.update({ + asset_id: srcAssetID + }, { + $set: { + asset_id: dstAssetID + } + }, { + multi: true + }); + + // Second remove the old asset. + await AssetModel.remove({ + id: srcAssetID + }); + + // That's it! } static all(skip = null, limit = null) { diff --git a/test/server/services/assets.js b/test/server/services/assets.js index e46e75576..748e177e0 100644 --- a/test/server/services/assets.js +++ b/test/server/services/assets.js @@ -1,22 +1,29 @@ const AssetModel = require('../../../models/asset'); +const CommentModel = require('../../../models/comment'); const AssetsService = require('../../../services/assets'); +const CommentsService = require('../../../services/comments'); const SettingsService = require('../../../services/settings'); +const url = require('url'); const chai = require('chai'); const expect = chai.expect; +const chaiAsPromised = require('chai-as-promised'); + +chai.use(chaiAsPromised); // Use the chai should. chai.should(); +const settings = {id: '1', moderation: 'PRE', domains: {whitelist: ['new.test.com', 'test.com', 'override.test.com']}}; +const defaults = {url:'http://test.com'}; + describe('services.AssetsService', () => { - beforeEach(() => { - const settings = {id: '1', moderation: 'PRE', domains: {whitelist: ['new.test.com', 'test.com', 'override.test.com']}}; - const defaults = {url:'http://test.com'}; + let asset; + beforeEach(async () => { + await SettingsService.init(settings); - return SettingsService.init(settings).then(() => { - return AssetModel.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}); - }); + asset = await AssetModel.findOneAndUpdate({id: '1'}, {$setOnInsert: defaults}, {upsert: true, new: true}); }); describe('#findById', ()=> { @@ -120,4 +127,79 @@ describe('services.AssetsService', () => { }); }); }); + + describe('#updateURL', () => { + + it('should change the url if the asset was found, and there was no conflict', async () => { + let newURL = url.resolve(asset.url, '/new-url'); + + // Update the asset. + await AssetsService.updateURL(asset.id, newURL); + + // Check that the url was updated. + let {url: databaseURL} = await AssetsService.findById(asset.id); + + expect(databaseURL).to.equal(newURL); + }); + + it('should error if the new url already exists', async () => { + let newURL = url.resolve(asset.url, '/new-url'); + + // Create a new asset with our new URL. + await AssetModel.findOneAndUpdate({id: '2'}, {$setOnInsert: {url: newURL}}, {upsert: true, new: true}); + + return AssetsService.updateURL(asset.id, newURL).should.eventually.be.rejected; + }); + + }); + + describe('#merge', () => { + + it('should error if either the src or the dst is missing', () => { + return AssetsService.merge('not-found', asset.id).should.eventually.be.rejected; + }); + + it('should merge the assets', async () => { + let newURL = url.resolve(asset.url, '/new-url'); + + // Create a new asset with our new URL. + await AssetModel.findOneAndUpdate({id: '2'}, {$setOnInsert: {url: newURL}}, {upsert: true, new: true}); + + // Create some comments on both assets. + await CommentsService.publicCreate([ + { + asset_id: '1', + body: 'This is a comment!', + status: 'ACCEPTED' + }, + { + asset_id: '1', + body: 'This is a comment!', + status: 'ACCEPTED' + }, + { + asset_id: '2', + body: 'This is a comment!', + status: 'ACCEPTED' + }, + { + asset_id: '2', + body: 'This is a comment!', + status: 'ACCEPTED' + } + ]); + + // Merge all the comments from asset 1 into asset 2, followed by deleting + // asset 1. + await AssetsService.merge('1', '2'); + + // Check to see if the comments are moved. + expect(await CommentModel.find({asset_id: '1'}).count()).to.equal(0); + expect(await CommentModel.find({asset_id: '2'}).count()).to.equal(4); + + // Check to see if the asset was removed. + expect(await AssetModel.findOne({id: '1'})).to.equal(null); + }); + + }); });