From 325162cb3ef40c07ca8ac8d083151b52aaf9d80d Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Tue, 22 Nov 2016 17:15:14 -0700 Subject: [PATCH 1/8] Added authorization pieces + test functions --- middleware/authorization.js | 44 +++++++++++++-------- models/action.js | 16 +++++++- routes/api/actions/index.js | 3 +- routes/api/asset/index.js | 11 ------ routes/api/auth/index.js | 2 +- routes/api/comments/index.js | 9 +++-- routes/api/index.js | 7 ++-- routes/api/queue/index.js | 4 +- routes/api/settings/index.js | 5 ++- routes/api/stream/index.js | 2 +- routes/api/user/index.js | 5 ++- tests/models/action.js | 61 +++++++++++++++++++++-------- tests/routes/api/assets/index.js | 63 +----------------------------- tests/routes/api/comments/index.js | 30 +++++++++++--- tests/routes/api/queue/index.js | 2 + tests/routes/api/settings/index.js | 54 +++++++++++++------------ tests/utils/passport.js | 25 ++++++++++++ 17 files changed, 190 insertions(+), 153 deletions(-) create mode 100644 tests/utils/passport.js diff --git a/middleware/authorization.js b/middleware/authorization.js index 238219a06..65f77b753 100644 --- a/middleware/authorization.js +++ b/middleware/authorization.js @@ -2,7 +2,9 @@ * authorization contains the references to the authorization middleware. * @type {Object} */ -const authorization = module.exports = {}; +const authorization = module.exports = { + middleware: [] +}; const debug = require('debug')('talk:middleware:authorization'); @@ -33,21 +35,29 @@ authorization.has = (user, ...roles) => roles.every((role) => user.roles.indexOf * @param {Array} roles all the roles that a user must have * @return {Callback} connect middleware */ -authorization.needed = (...roles) => (req, res, next) => { - // All routes that are wrapepd with this middleware actually require a role. - if (!req.user) { - debug(`No user on request, returning with ${ErrNotAuthorized}`); - return next(ErrNotAuthorized); - } +authorization.needed = (...roles) => [ - // Check to see if the current user has all the roles requested for the given - // array of roles requested, if one is not on the user, then this will - // evaluate to true. - if (!authorization.has(req.user, ...roles)) { - debug('User does not have all the required roles to access this page'); - return next(ErrNotAuthorized); - } + // Insert the pre-needed middlware. + ...authorization.middleware, - // Looks like they're allowed! - return next(); -}; + // Insert the actual middleware to check for the required role. + (req, res, next) => { + + // All routes that are wrapepd with this middleware actually require a role. + if (!req.user) { + debug(`No user on request, returning with ${ErrNotAuthorized}`); + return next(ErrNotAuthorized); + } + + // Check to see if the current user has all the roles requested for the given + // array of roles requested, if one is not on the user, then this will + // evaluate to true. + if (!authorization.has(req.user, ...roles)) { + debug('User does not have all the required roles to access this page'); + return next(ErrNotAuthorized); + } + + // Looks like they're allowed! + return next(); + } +]; diff --git a/models/action.js b/models/action.js index 91378b0b0..48c0a9a18 100644 --- a/models/action.js +++ b/models/action.js @@ -41,7 +41,7 @@ ActionSchema.statics.findByItemIdArray = function(item_ids) { * Returns summaries of actions for an array of ids * @param {String} ids array of user identifiers (uuid) */ -ActionSchema.statics.getActionSummaries = function(item_ids) { +ActionSchema.statics.getActionSummaries = function(item_ids, current_user_id = '') { return Action.aggregate([ { @@ -71,6 +71,18 @@ ActionSchema.statics.getActionSummaries = function(item_ids) { // just grabbing the last instance of the item type here. item_type: { $last: '$item_type' + }, + + current_user: { + $max: { + $cond: { + if: { + $eq: ['$user_id', current_user_id], + }, + then: '$$CURRENT', + else: null + } + } } } }, @@ -89,7 +101,7 @@ ActionSchema.statics.getActionSummaries = function(item_ids) { item_type: '$item_type', // set the current user to false here - current_user: {$literal: false} + current_user: '$current_user' } } ]) diff --git a/routes/api/actions/index.js b/routes/api/actions/index.js index f2f9be390..9724f5a73 100644 --- a/routes/api/actions/index.js +++ b/routes/api/actions/index.js @@ -6,7 +6,8 @@ const router = express.Router(); router.delete('/:action_id', (req, res, next) => { Action .findOneAndRemove({ - id: req.params.action_id + id: req.params.action_id, + user_id: req.user.id }) .then(() => { res.status(204).end(); diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js index b261da883..217c49547 100644 --- a/routes/api/asset/index.js +++ b/routes/api/asset/index.js @@ -30,15 +30,4 @@ router.get('/:id', (req, res, next) => { }); -// Upsert an asset and return the affected document. -router.put('/', (req, res, next) => { - - Asset.upsert(req.body) - .then((asset) => { - res.json(asset); - }) - .catch(next); - -}); - module.exports = router; diff --git a/routes/api/auth/index.js b/routes/api/auth/index.js index ae6fba01c..b88b6fe12 100644 --- a/routes/api/auth/index.js +++ b/routes/api/auth/index.js @@ -14,7 +14,7 @@ router.get('/', authorization.needed(), (req, res) => { /** * This destroys the session of a user, if they have one. */ -router.delete('/', (req, res) => { +router.delete('/', authorization.needed(), (req, res) => { req.session.destroy(() => { res.status(204).end(); }); diff --git a/routes/api/comments/index.js b/routes/api/comments/index.js index d318857d6..1507e82aa 100644 --- a/routes/api/comments/index.js +++ b/routes/api/comments/index.js @@ -1,10 +1,11 @@ const express = require('express'); const Comment = require('../../../models/comment'); const wordlist = require('../../../services/wordlist'); +const authorization = require('../../../middleware/authorization'); const router = express.Router(); -router.get('/', (req, res, next) => { +router.get('/', authorization.needed('admin'), (req, res, next) => { let query; if (req.query.status) { @@ -49,7 +50,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { }); }); -router.get('/:comment_id', (req, res, next) => { +router.get('/:comment_id', authorization.needed('admin'), (req, res, next) => { Comment .findById(req.params.comment_id) .then(comment => { @@ -65,7 +66,7 @@ router.get('/:comment_id', (req, res, next) => { }); }); -router.delete('/:comment_id', (req, res, next) => { +router.delete('/:comment_id', authorization.needed('admin'), (req, res, next) => { Comment .removeById(req.params.comment_id) .then(() => { @@ -76,7 +77,7 @@ router.delete('/:comment_id', (req, res, next) => { }); }); -router.put('/:comment_id/status', (req, res, next) => { +router.put('/:comment_id/status', authorization.needed('admin'), (req, res, next) => { const { status diff --git a/routes/api/index.js b/routes/api/index.js index 7192a1324..8c6014ed8 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -1,14 +1,15 @@ const express = require('express'); +const authorization = require('../../middleware/authorization'); const router = express.Router(); router.use('/asset', require('./asset')); router.use('/auth', require('./auth')); -router.use('/comments', require('./comments')); +router.use('/comments', authorization.needed(), require('./comments')); router.use('/queue', require('./queue')); -router.use('/settings', require('./settings')); +router.use('/settings', authorization.needed('admin'), require('./settings')); router.use('/stream', require('./stream')); router.use('/user', require('./user')); -router.use('/actions', require('./actions')); +router.use('/actions', authorization.needed(), require('./actions')); module.exports = router; diff --git a/routes/api/queue/index.js b/routes/api/queue/index.js index f8ab95d6b..edc5602ef 100644 --- a/routes/api/queue/index.js +++ b/routes/api/queue/index.js @@ -1,7 +1,7 @@ const express = require('express'); const Comment = require('../../../models/comment'); - const Setting = require('../../../models/setting'); +const authorization = require('../../../middleware/authorization'); const router = express.Router(); @@ -13,7 +13,7 @@ const router = express.Router(); // depending on the settings. The :moderation overwrites this settings. // Pre-moderation: New comments are shown in the moderator queues immediately. // Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users. -router.get('/comments/pending', (req, res, next) => { +router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => { Setting.getModerationSetting().then(function({moderation}){ Comment.moderationQueue(moderation).then((comments) => { res.status(200).json(comments); diff --git a/routes/api/settings/index.js b/routes/api/settings/index.js index 2665cacc8..6e8c64d4f 100644 --- a/routes/api/settings/index.js +++ b/routes/api/settings/index.js @@ -1,7 +1,8 @@ -const _ = require('lodash'); const express = require('express'); -const router = express.Router(); const Setting = require('../../../models/setting'); +const _ = require('lodash'); + +const router = express.Router(); router.get('/', (req, res, next) => { Setting diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index acbfe3d77..a48548608 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -41,7 +41,7 @@ router.get('/', (req, res, next) => { asset.id, ...comments.map((comment) => comment.id), ...comments.map((comment) => comment.author_id) - ])) + ]), req.user ? req.user.id : '') ]); }) .then(([assets, comments, users, actions]) => { diff --git a/routes/api/user/index.js b/routes/api/user/index.js index 3d5b49d93..3251e0608 100644 --- a/routes/api/user/index.js +++ b/routes/api/user/index.js @@ -7,8 +7,9 @@ const fs = require('fs'); const path = require('path'); const resetEmailFile = fs.readFileSync(path.resolve(__dirname, '../../../views/password-reset-email.ejs')); const resetEmailTemplate = ejs.compile(resetEmailFile.toString()); +const authorization = require('../../../middleware/authorization'); -router.get('/', (req, res, next) => { +router.get('/', authorization.needed('admin'), (req, res, next) => { const { value = '', field = 'created_at', @@ -49,7 +50,7 @@ router.get('/', (req, res, next) => { .catch(next); }); -router.post('/:user_id/role', (req, res, next) => { +router.post('/:user_id/role', authorization.needed('admin'), (req, res, next) => { User .addRoleToUser(req.params.user_id, req.body.role) .then(role => { diff --git a/tests/models/action.js b/tests/models/action.js index 78f5fd1d0..87d53b7da 100644 --- a/tests/models/action.js +++ b/tests/models/action.js @@ -5,23 +5,25 @@ const expect = require('chai').expect; describe('Action: models', () => { let mockActions; + beforeEach(() => { return Action.create([{ action_type: 'flag', item_id: '123', - item_type: 'comments' + item_type: 'comment', + user_id: 'flagginguserid' }, { action_type: 'flag', item_id: '456', - item_type: 'comments' + item_type: 'comment' }, { action_type: 'flag', item_id: '123', - item_type: 'comments' + item_type: 'comment' }, { action_type: 'like', item_id: '123', - item_type: 'comments' + item_type: 'comment' }]).then((actions) => { mockActions = actions; }); @@ -30,8 +32,7 @@ describe('Action: models', () => { describe('#findById()', () => { it('should find an action by id', () => { return Action.findById(mockActions[0].id).then((result) => { - expect(result).to.have.property('action_type') - .and.to.equal('flag'); + expect(result).to.have.property('action_type', 'flag'); }); }); }); @@ -46,27 +47,55 @@ describe('Action: models', () => { describe('#getActionSummaries()', () => { it('should return properly formatted summaries from an array of item_ids', () => { - return Action.getActionSummaries(['123', '789']).then((result) => { - expect(result).to.have.length(2); + return Action.getActionSummaries(['123', '789']).then((summaries) => { + expect(summaries).to.have.length(2); - const sorted = result.sort((a, b) => a.count - b.count); - - expect(sorted[0]).to.deep.equal({ + expect(summaries).to.deep.include({ action_type: 'like', count: 1, item_id: '123', - item_type: 'comments', - current_user: false + item_type: 'comment', + current_user: null }); - expect(sorted[1]).to.deep.equal({ + expect(summaries).to.deep.include({ action_type: 'flag', count: 2, item_id: '123', - item_type: 'comments', - current_user: false + item_type: 'comment', + current_user: null }); }); }); + + it('should include a current user when one is passed', () => { + return Action + .getActionSummaries(['123'], 'flagginguserid') + .then((summaries) => { + expect(summaries).to.have.length(2); + + let summary = summaries.find((s) => s.item_id === '123' && s.action_type === 'flag'); + + expect(summary).to.not.be.undefined; + expect(summary.current_user).to.not.be.null; + expect(summary.current_user).to.have.property('item_id', '123'); + expect(summary.current_user).to.have.property('item_type', 'comment'); + expect(summary.current_user).to.have.property('user_id', 'flagginguserid'); + expect(summary.current_user).to.have.property('action_type', 'flag'); + }); + }); + + it('should not include a current user when one is passed for a user that doesn\'t have an action', () => { + return Action + .getActionSummaries(['123'], 'flagginguserid2') + .then((summaries) => { + expect(summaries).to.have.length(2); + + summaries.forEach((summary) => { + expect(summary).to.not.be.undefined; + expect(summary).to.have.property('current_user', null); + }); + }); + }); }); }); diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js index aa764e214..542e88c42 100644 --- a/tests/routes/api/assets/index.js +++ b/tests/routes/api/assets/index.js @@ -1,22 +1,13 @@ require('../../../utils/mongoose'); +const passport = require('../../../utils/passport'); const chai = require('chai'); -const expect = chai.expect; const server = require('../../../../app'); // Setup chai. chai.should(); chai.use(require('chai-http')); -let fixture = { - 'url': 'http://hhgg.com/total-perspective-vortex', - 'type': 'article', - 'headline': 'The Total Perspective Vortex', - 'summary': 'You are an insignificant dot on an insignificant dot.', - 'section': 'Everything', - 'authors': ['Ford Prefect'] -}; - describe('Asset: routes', () => { describe('/GET Asset', () => { @@ -25,6 +16,7 @@ describe('Asset: routes', () => { chai.request(server) .get('/api/v1/asset') + .set(passport.inject({roles: ['admin']})) .end((err, res) => { if (err) { @@ -41,55 +33,4 @@ describe('Asset: routes', () => { }); }); - // This test checks PUT and read - describe('/PUT Asset', () => { - describe('#put', () => { - it('It should save an asset and load it again.', (done) => { - - chai.request(server) - .put('/api/v1/asset') - .send(fixture) - .end((err, res) => { - - if (err) { - throw new Error(err); - } - - res.should.have.status(200); - res.body.should.be.a('object'); - - // Id should be generated by the model if absent. - res.body.should.have.property('id'); - - // Save the asset id to compare with GET result. - let assetId = res.body.id; - - // Load the asset to make sure it's really there. - chai.request(server) - .get(`/api/v1/asset?url=${encodeURIComponent(fixture.url)}`) - .end((err, res) => { - - if (err) { - throw new Error(err); - } - - res.should.have.status(200); - res.body.should.be.an('array'); - - let asset = res.body[0]; - - expect(asset).to.have.property('id'); - - // Ensure the asset has the same id as above. - // This tests the single url per Id concept. - expect(assetId).to.equal(asset.id); - - done(); - - }); - }); - }); - }); - }); // End describe /PUT Asset - }); diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index ec9e05d03..be3d14f69 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -1,6 +1,7 @@ process.env.NODE_ENV = 'test'; require('../../../utils/mongoose'); +const passport = require('../../../utils/passport'); const app = require('../../../../app'); const chai = require('chai'); @@ -68,6 +69,7 @@ describe('Get /comments', () => { it('should return all the comments', () => { return chai.request(app) .get('/api/v1/comments') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); @@ -126,6 +128,7 @@ describe('Get comments by status and action', () => { it('should return all the rejected comments', () => { return chai.request(app) .get('/api/v1/comments?status=rejected') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); expect(res.body[0]).to.have.property('id', 'abc'); @@ -135,6 +138,7 @@ describe('Get comments by status and action', () => { it('should return all the approved comments', () => { return chai.request(app) .get('/api/v1/comments?status=accepted') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); expect(res.body[0]).to.have.property('id', 'hij'); @@ -144,6 +148,7 @@ describe('Get comments by status and action', () => { it('should return all the new comments', () => { return chai.request(app) .get('/api/v1/comments?status=new') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); expect(res.body[0]).to.have.property('id', 'def'); @@ -153,6 +158,7 @@ describe('Get comments by status and action', () => { it('should return all the flagged comments', () => { return chai.request(app) .get('/api/v1/comments?action_type=flag') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); @@ -195,6 +201,7 @@ describe('Post /comments', () => { it('should create a comment', () => { return chai.request(app) .post('/api/v1/comments') + .set(passport.inject({roles: []})) .send({'body': 'Something body.', 'author_id': '123', 'asset_id': '1', 'parent_id': ''}) .then((res) => { expect(res).to.have.status(201); @@ -205,6 +212,7 @@ describe('Post /comments', () => { it('should create a comment with a rejected status if it contains a bad word', () => { return chai.request(app) .post('/api/v1/comments') + .set(passport.inject({roles: []})) .send({'body': 'bad words are the baddest', 'author_id': '123', 'asset_id': '1', 'parent_id': ''}) .then((res) => { expect(res).to.have.status(201); @@ -262,6 +270,7 @@ describe('Get /:comment_id', () => { it('should return the right comment for the comment_id', () => { return chai.request(app) .get('/api/v1/comments/abc') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(200); expect(res).to.have.property('body'); @@ -318,6 +327,7 @@ describe('Remove /:comment_id', () => { it('it should remove comment', () => { return chai.request(app) .delete('/api/v1/comments/abc') + .set(passport.inject({roles: ['admin']})) .then((res) => { expect(res).to.have.status(204); @@ -329,11 +339,6 @@ describe('Remove /:comment_id', () => { }); }); -process.on('unhandledRejection', (reason) => { - console.error('Reason: '); - console.error(reason); -}); - describe('Put /:comment_id/status', () => { const comments = [{ @@ -384,12 +389,26 @@ describe('Put /:comment_id/status', () => { it('it should update status', function() { return chai.request(app) .put('/api/v1/comments/abc/status') + .set(passport.inject({roles: ['admin']})) .send({status: 'accepted'}) .then((res) => { expect(res).to.have.status(204); expect(res.body).to.be.empty; }); }); + + it('it should not allow a non-admin to update status', () => { + return chai.request(app) + .put('/api/v1/comments/abc/status') + .set(passport.inject({roles: []})) + .send({status: 'accepted'}) + .then((res) => { + expect(res).to.be.empty; + }) + .catch((err) => { + expect(err).to.have.property('status', 401); + }); + }); }); describe('Post /:comment_id/actions', () => { @@ -442,6 +461,7 @@ describe('Post /:comment_id/actions', () => { it('it should update actions', () => { return chai.request(app) .post('/api/v1/comments/abc/actions') + .set(passport.inject({roles: ['admin']})) .send({'user_id': '456', 'action_type': 'flag'}) .then((res) => { expect(res).to.have.status(201); diff --git a/tests/routes/api/queue/index.js b/tests/routes/api/queue/index.js index f21fe1331..4eff3d384 100644 --- a/tests/routes/api/queue/index.js +++ b/tests/routes/api/queue/index.js @@ -1,6 +1,7 @@ process.env.NODE_ENV = 'test'; require('../../../utils/mongoose'); +const passport = require('../../../utils/passport'); const app = require('../../../../app'); const chai = require('chai'); @@ -71,6 +72,7 @@ describe('Get moderation queues rejected, pending, flags', () => { it('should return all the pending comments', function(done){ chai.request(app) .get('/api/v1/queue/comments/pending') + .set(passport.inject({roles: ['admin']})) .end(function(err, res){ expect(err).to.be.null; expect(res).to.have.status(200); diff --git a/tests/routes/api/settings/index.js b/tests/routes/api/settings/index.js index 41c37828e..86feca1eb 100644 --- a/tests/routes/api/settings/index.js +++ b/tests/routes/api/settings/index.js @@ -1,13 +1,15 @@ process.env.NODE_ENV = 'test'; require('../../../utils/mongoose'); +const passport = require('../../../utils/passport'); const app = require('../../../../app'); const chai = require('chai'); -const chaiHttp = require('chai-http'); -chai.use(chaiHttp); const expect = chai.expect; +chai.should(); +chai.use(require('chai-http')); + const Setting = require('../../../../models/setting'); const defaults = {id: '1', moderation: 'pre'}; @@ -17,15 +19,16 @@ describe('GET /settings', () => { return Setting.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}); }); - it('should return a settings object', done => { - chai.request(app) + it('should return a settings object', () => { + return chai.request(app) .get('/api/v1/settings') - .end((err, res) => { - expect(err).to.be.null; + .set(passport.inject({ + roles: ['admin'] + })) + .then((res) => { expect(res).to.have.status(200); expect(res).to.be.json; expect(res.body).to.have.property('moderation', 'pre'); - done(err); }); }); }); @@ -33,25 +36,26 @@ describe('GET /settings', () => { // update the settings. describe('update settings', () => { it('should respond ok to a PUT', () => { - return Setting.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}) - .then(() => { - return chai.request(app) - .put('/api/v1/settings') - .send({moderation: 'post'}) - .then(res => { - expect(res).to.have.status(204); + return Setting + .update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}) + .then(() => { + return chai.request(app) + .put('/api/v1/settings') + .set(passport.inject({ + roles: ['admin'] + })) + .send({moderation: 'post'}); + }) + .then(res => { + expect(res).to.have.status(204); - return Setting.getSettings(); + return Setting.getSettings(); + }) + .then(settings => { - }) - .then(settings => { - // confirm updated settings in db - expect(settings).to.have.property('moderation'); - expect(settings.moderation).to.equal('post'); - }) - .catch(err => { - throw err; - }); - }); + // confirm updated settings in db + expect(settings).to.have.property('moderation'); + expect(settings.moderation).to.equal('post'); + }); }); }); diff --git a/tests/utils/passport.js b/tests/utils/passport.js new file mode 100644 index 000000000..401e50b77 --- /dev/null +++ b/tests/utils/passport.js @@ -0,0 +1,25 @@ +const authorization = require('../../middleware/authorization'); + +// Add the passport middleware here before it's setup. +authorization.middleware.push((req, res, next) => { + req.user = JSON.parse(new Buffer(req.get('X-Mock-Authorization'), 'base64').toString('ascii')); + + next(); +}); + +const MockStrategy = { + + /** + * Injects the new user into the request header for the mock middleware to + * interpret. + * @param {Object} user the user to inject + * @return {Object} the headers to add to the request + */ + inject(user) { + return { + 'X-Mock-Authorization': new Buffer(JSON.stringify(user)).toString('base64') + }; + } +}; + +module.exports = MockStrategy; From 7a6ea34468a4356a53e2f94c310a0fc0da3cc843 Mon Sep 17 00:00:00 2001 From: David Erwin Date: Fri, 25 Nov 2016 10:42:38 -0500 Subject: [PATCH 2/8] Remove the preview link --- routes/admin/index.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/routes/admin/index.js b/routes/admin/index.js index 9d3cbd0a9..2931626be 100644 --- a/routes/admin/index.js +++ b/routes/admin/index.js @@ -1,12 +1,8 @@ const express = require('express'); const router = express.Router(); -router.get('/embed/stream/preview', (req, res) => { - res.render('embed-stream', {basePath: '/client/embed/stream'}); -}); - -// this route is expecting there to be a token in the hash -// see /views/password-reset-email.ejs +// GET /password-reset expects an OpenID token in the hash. +// Links to this endpoit are generated in /views/password-reset-email.ejs. router.get('/password-reset', (req, res, next) => { // TODO: store the redirect uri in the token or something fancy // admins and regular users should probably be redirected to different places. From 93cd4ea74c7a9654319f63ea9da4c8eb3c6da8d9 Mon Sep 17 00:00:00 2001 From: David Erwin Date: Fri, 25 Nov 2016 10:43:32 -0500 Subject: [PATCH 3/8] Remove non-standard stream embed template --- views/embed/stream.ejs | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 views/embed/stream.ejs diff --git a/views/embed/stream.ejs b/views/embed/stream.ejs deleted file mode 100644 index 5cdef0811..000000000 --- a/views/embed/stream.ejs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - -
- - - From a022ef8a52fa53dd74af5e562da9d4efe32d1dc4 Mon Sep 17 00:00:00 2001 From: David Erwin Date: Fri, 25 Nov 2016 10:50:30 -0500 Subject: [PATCH 4/8] Restore embed/stream.ejs --- views/embed/stream.ejs | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 views/embed/stream.ejs diff --git a/views/embed/stream.ejs b/views/embed/stream.ejs new file mode 100644 index 000000000..f87318db6 --- /dev/null +++ b/views/embed/stream.ejs @@ -0,0 +1,13 @@ + + + + + + + + +
+ + + \ No newline at end of file From cb31caed4b0378a3d9fc5784e3f0b6cd53604ef1 Mon Sep 17 00:00:00 2001 From: David Erwin Date: Fri, 25 Nov 2016 11:47:00 -0500 Subject: [PATCH 5/8] Update comments --- routes/admin/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routes/admin/index.js b/routes/admin/index.js index 2931626be..03852c375 100644 --- a/routes/admin/index.js +++ b/routes/admin/index.js @@ -1,10 +1,10 @@ const express = require('express'); const router = express.Router(); -// GET /password-reset expects an OpenID token in the hash. -// Links to this endpoit are generated in /views/password-reset-email.ejs. +// Get /password-reset expects a signed token (JWT) in the hash. +// Links to this endpoint are generated by /views/password-reset-email.ejs. router.get('/password-reset', (req, res, next) => { - // TODO: store the redirect uri in the token or something fancy + // TODO: store the redirect uri in the token or something fancy. // admins and regular users should probably be redirected to different places. res.render('password-reset', {redirectUri: process.env.TALK_ROOT_URL}); }); From 9d8968f1ce37e332cc979291d4c61dd9b64e2dcc Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 28 Nov 2016 10:14:13 -0700 Subject: [PATCH 6/8] Corrected action upserting functionality --- models/comment.js | 25 ++++++++++++++++++------- routes/api/comments/index.js | 8 +++----- tests/routes/api/comments/index.js | 2 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/models/comment.js b/models/comment.js index 2ac130978..73f8250d7 100644 --- a/models/comment.js +++ b/models/comment.js @@ -161,15 +161,26 @@ CommentSchema.statics.changeStatus = function(id, status) { * @param {String} action the new action to the comment * @return {Promise} */ -CommentSchema.statics.addAction = function(id, user_id, action_type) { - // check that the comment exist - let action = new Action({ - action_type: action_type, +CommentSchema.statics.addAction = function(item_id, user_id, action_type) { + const action = { + item_id, item_type: 'comment', - item_id: id, - user_id: user_id + user_id, + action_type + }; + + // Update/Create the action for the user. + return Action.findOneAndUpdate(action, action, { + + // Ensure that if it's new, we return the new object created. + new: true, + + // Perform an upsert in the event that this doesn't exist. + upsert: true, + + // Set the default values if not provided based on the mongoose models. + setDefaultsOnInsert: true }); - return action.save(); }; //============================================================================== diff --git a/routes/api/comments/index.js b/routes/api/comments/index.js index 1507e82aa..bfb3d67ab 100644 --- a/routes/api/comments/index.js +++ b/routes/api/comments/index.js @@ -29,8 +29,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { const { body, asset_id, - parent_id, - author_id + parent_id } = req.body; Comment @@ -39,7 +38,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => { asset_id, parent_id, status: req.wordlist.matched ? 'rejected' : '', - author_id + author_id: req.user.id }) .then((comment) => { @@ -96,12 +95,11 @@ router.put('/:comment_id/status', authorization.needed('admin'), (req, res, next router.post('/:comment_id/actions', (req, res, next) => { const { - user_id, action_type } = req.body; Comment - .addAction(req.params.comment_id, user_id, action_type) + .addAction(req.params.comment_id, req.user.id, action_type) .then((action) => { res.status(201).json(action); }) diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index be3d14f69..29775d3d0 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -461,7 +461,7 @@ describe('Post /:comment_id/actions', () => { it('it should update actions', () => { return chai.request(app) .post('/api/v1/comments/abc/actions') - .set(passport.inject({roles: ['admin']})) + .set(passport.inject({id: '456', roles: ['admin']})) .send({'user_id': '456', 'action_type': 'flag'}) .then((res) => { expect(res).to.have.status(201); From d34cbc7870206717c66f461bdb24bbfca8718980 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 28 Nov 2016 10:19:33 -0700 Subject: [PATCH 7/8] Moved the action insert into the action model. --- models/action.js | 29 +++++++++++++++++++++++++++++ models/comment.js | 30 ++++++++---------------------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/models/action.js b/models/action.js index 48c0a9a18..891dbd4f5 100644 --- a/models/action.js +++ b/models/action.js @@ -27,6 +27,35 @@ ActionSchema.statics.findById = function(id) { return Action.findOne({id}); }; +/** + * Add an action. + * @param {String} item_id identifier of the comment (uuid) + * @param {String} user_id user id of the action (uuid) + * @param {String} action the new action to the comment + * @return {Promise} + */ +ActionSchema.statics.insertUserAction = ({item_id, item_type, user_id, action_type}) => { + const action = { + item_id, + item_type, + user_id, + action_type + }; + + // Create/Update the action. + return Action.findOneAndUpdate(action, action, { + + // Ensure that if it's new, we return the new object created. + new: true, + + // Perform an upsert in the event that this doesn't exist. + upsert: true, + + // Set the default values if not provided based on the mongoose models. + setDefaultsOnInsert: true + }); +}; + /** * Finds actions in an array of ids. * @param {String} ids array of user identifiers (uuid) diff --git a/models/comment.js b/models/comment.js index 73f8250d7..c1a32cf1b 100644 --- a/models/comment.js +++ b/models/comment.js @@ -157,31 +157,17 @@ CommentSchema.statics.changeStatus = function(id, status) { /** * Add an action to the comment. - * @param {String} id identifier of the comment (uuid) + * @param {String} item_id identifier of the comment (uuid) + * @param {String} user_id user id of the action (uuid) * @param {String} action the new action to the comment * @return {Promise} */ -CommentSchema.statics.addAction = function(item_id, user_id, action_type) { - const action = { - item_id, - item_type: 'comment', - user_id, - action_type - }; - - // Update/Create the action for the user. - return Action.findOneAndUpdate(action, action, { - - // Ensure that if it's new, we return the new object created. - new: true, - - // Perform an upsert in the event that this doesn't exist. - upsert: true, - - // Set the default values if not provided based on the mongoose models. - setDefaultsOnInsert: true - }); -}; +CommentSchema.statics.addAction = (item_id, user_id, action_type) => Action.insertUserAction({ + item_id, + item_type: 'comment', + user_id, + action_type +}); //============================================================================== // Remove Statics From b0f01cf00f73f986c74144dfb7f92039e79b4fa7 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 28 Nov 2016 11:29:39 -0700 Subject: [PATCH 8/8] Initial commit of asset queuing (#97) * Initial commit of asset queuing * Addresssing comments --- .eslintrc.json | 41 ++--- bin/cli | 3 + bin/cli-assets | 76 ++++++++++ bin/cli-jobs | 49 ++++++ bin/cli-serve | 132 +++++++++++++++++ bin/cli-settings | 15 +- bin/cli-users | 85 ++++------- bin/www | 97 ------------ .../coral-embed-stream/src/CommentStream.js | 8 +- client/coral-framework/actions/auth.js | 8 +- models/asset.js | 89 ++++------- package.json | 12 +- routes/api/asset/index.js | 78 ++++++++-- routes/api/auth/index.js | 13 +- routes/api/index.js | 17 ++- routes/api/queue/index.js | 3 +- routes/api/stream/index.js | 91 +++++++++--- routes/api/user/index.js | 5 +- routes/index.js | 4 +- services/scraper.js | 140 ++++++++++++++++++ swagger.yaml | 87 +++++++++++ tests/index.js | 9 -- tests/models/action.js | 2 - tests/models/asset.js | 35 ----- tests/models/comment.js | 2 - tests/models/setting.js | 4 - tests/models/user.js | 2 - tests/{utils => }/mongoose.js | 6 +- tests/{utils => }/passport.js | 2 +- tests/routes/api/assets/index.js | 34 +---- tests/routes/api/auth/index.js | 2 - tests/routes/api/comments/index.js | 5 +- tests/routes/api/queue/index.js | 5 +- tests/routes/api/settings/index.js | 5 +- tests/routes/api/stream/index.js | 6 +- tests/services/scraper.js | 22 +++ util.js | 42 ++++++ views/article.ejs | 4 +- 38 files changed, 832 insertions(+), 408 deletions(-) create mode 100755 bin/cli-assets create mode 100755 bin/cli-jobs create mode 100755 bin/cli-serve delete mode 100755 bin/www create mode 100644 services/scraper.js delete mode 100644 tests/index.js rename tests/{utils => }/mongoose.js (71%) rename tests/{utils => }/passport.js (90%) create mode 100644 tests/services/scraper.js create mode 100644 util.js diff --git a/.eslintrc.json b/.eslintrc.json index 6ac5a08e6..035a86189 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,25 +5,15 @@ }, "extends": "eslint:recommended", "rules": { - "indent": [ - "error", + "indent": ["error", 2 ], "no-console": [ 0 ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "single" - ], - "semi": [ - "error", - "always" - ], + "linebreak-style": ["error", "unix"], + "quotes": ["error", "single"], + "semi": ["error", "always"], "no-template-curly-in-string": [1], "no-unsafe-negation": [1], "array-callback-return": [1], @@ -35,7 +25,6 @@ "no-throw-literal": [2], "yoda": [1], "no-path-concat": [2], - "no-process-exit": [2], "eol-last": [1], "no-continue": [1], "no-nested-ternary": [1], @@ -46,20 +35,20 @@ "no-const-assign": [2], "no-duplicate-imports": [2], "prefer-template": [1], - "comma-spacing": [ - "error", - { + "comma-spacing": ["error", { "after": true - } - ], + }], "no-var": [2], "no-lonely-if": [2], "curly": [2], - "no-unused-vars": ["error", { "argsIgnorePattern": "next" }], - "no-multiple-empty-lines": [ - "error", - {"max": 1} - ], - "newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }] + "no-unused-vars": ["error", { + "argsIgnorePattern": "next" + }], + "no-multiple-empty-lines": ["error", { + "max": 1 + }], + "newline-per-chained-call": ["error", { + "ignoreChainWithDepth": 2 + }] } } diff --git a/bin/cli b/bin/cli index bec5ea84f..d8148282f 100755 --- a/bin/cli +++ b/bin/cli @@ -19,7 +19,10 @@ const pkg = require('../package.json'); program .version(pkg.version) + .command('serve', 'serve the application') + .command('assets', 'interact with assets') .command('settings', 'work with the application settings') + .command('jobs', 'work with the job queues') .command('users', 'work with the application auth') .parse(process.argv); diff --git a/bin/cli-assets b/bin/cli-assets new file mode 100755 index 000000000..99965c6d9 --- /dev/null +++ b/bin/cli-assets @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const pkg = require('../package.json'); +const Table = require('cli-table'); +const Asset = require('../models/asset'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +// Register the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); + +/** + * Lists all the assets registered in the database. + */ +function listAssets() { + Asset + .find({}) + .sort({'created_at': 1}) + .then((asset) => { + let table = new Table({ + head: [ + 'ID', + 'Title', + 'URL' + ] + }); + + asset.forEach((asset) => { + table.push([ + asset.id, + asset.title ? asset.title : '', + asset.url ? asset.url : '' + ]); + }); + + console.log(table.toString()); + util.shutdown(); + }) + .catch((err) => { + console.error(err); + util.shutdown(1); + }); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .version(pkg.version); + +program + .command('list') + .description('list all the assets in the database') + .action(listAssets); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (!process.argv.slice(2).length) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-jobs b/bin/cli-jobs new file mode 100755 index 000000000..f14276d38 --- /dev/null +++ b/bin/cli-jobs @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const scraper = require('../services/scraper'); +const util = require('../util'); +const mongoose = require('../mongoose'); + +util.onshutdown([ + () => mongoose.disconnect() +]); + +function processJobs() { + + // Start the processor. + scraper.process(); + + // The scraper only needs to shutdown when the scraper has actually been + // started. + util.onshutdown([ + () => scraper.shutdown() + ]); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .command('process') + .description('starts job processing') + .action(processJobs); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (process.argv.length <= 2) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-serve b/bin/cli-serve new file mode 100755 index 000000000..14c1261d7 --- /dev/null +++ b/bin/cli-serve @@ -0,0 +1,132 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +const app = require('../app'); +const debug = require('debug')('talk:server'); +const http = require('http'); +const init = require('../init'); +const scraper = require('../services/scraper'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +/** +* Get port from environment and store in Express. +*/ +const port = normalizePort(process.env.TALK_PORT || '3000'); + +app.set('port', port); + +/** +* Create HTTP server. +*/ +const server = http.createServer(app); + +/** + * Event listener for HTTP server "error" event. + */ +function onError(error) { + if (error.syscall !== 'listen') { + throw error; + } + + let bind = typeof port === 'string' + ? `Pipe ${port}` + : `Port ${port}`; + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + console.error(`${bind} requires elevated privileges`); + break; + case 'EADDRINUSE': + console.error(`${bind} is already in use`); + break; + } + + throw error; +} + +/** + * Normalize a port into a number, string, or false. + */ + +function normalizePort(val) { + let port = parseInt(val, 10); + + if (isNaN(port)) { + // named pipe + return val; + } + + if (port >= 0) { + // port number + return port; + } + + return false; +} + +/** + * Event listener for HTTP server "listening" event. + */ + +function onListening() { + let addr = server.address(); + let bind = typeof addr === 'string' + ? `pipe ${ addr}` + : `port ${ addr.port}`; + debug(`Listening on ${ bind}`); +} + +/** + * Start the app. + */ +function startApp() { + init().then(() => { + + /** + * Listen on provided port, on all network interfaces. + */ + server.listen(port); + server.on('error', onError); + server.on('listening', onListening); + }); +} + +/** + * Module dependencies. + */ + +const program = require('commander'); + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .option('-j, --jobs', 'enable job processing on this thread') + .parse(process.argv); + +// Start the application serving. +startApp(); + +// Enable job processing on the thread if enabled. +if (program.jobs) { + + // Start the processor. + scraper.process(); +} + +// Define a safe shutdown function to call in the event we need to shutdown +// because the node hooks are below which will interrupt the shutdown process. +// Shutdown the mongoose connection, the app server, and the scraper. +util.onshutdown([ + () => program.jobs ? scraper.shutdown() : null, + () => mongoose.disconnect(), + () => server.close() +]); diff --git a/bin/cli-settings b/bin/cli-settings index cff6c04ed..c639a5ca5 100755 --- a/bin/cli-settings +++ b/bin/cli-settings @@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG; */ const program = require('commander'); +const mongoose = require('../mongoose'); +const Setting = require('../models/setting'); +const util = require('../util'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); //============================================================================== // Setting up the program command line arguments. @@ -20,19 +28,17 @@ program .command('init') .description('initilizes the talk settings') .action(() => { - const mongoose = require('../mongoose'); - const Setting = require('../models/setting'); const defaults = {id: '1', moderation: 'pre'}; Setting .update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}) .then(() => { console.log('Created settings object.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(`failed to create the settings object ${JSON.stringify(err)}`); - throw new Error(err); // just to be safe + util.shutdown(1); }); }); @@ -41,4 +47,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/cli-users b/bin/cli-users index 54d7f2927..2e0dc84e6 100755 --- a/bin/cli-users +++ b/bin/cli-users @@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG; const program = require('commander'); const pkg = require('../package.json'); const prompt = require('prompt'); +const User = require('../models/user'); +const mongoose = require('../mongoose'); +const util = require('../util'); +const Table = require('cli-table'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); /** * Prompts for input and registers a user based on those. */ function createUser(options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - return new Promise((resolve, reject) => { if (options.flag_mode) { @@ -74,11 +80,11 @@ function createUser(options) { }) .then((user) => { console.log(`Created user ${user.id}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -86,20 +92,17 @@ function createUser(options) { * Deletes a user. */ function deleteUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .findOneAndRemove({ id: userID }) .then(() => { console.log('Deleted user'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -107,9 +110,6 @@ function deleteUser(userID) { * Changes the password for a user. */ function passwd(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - prompt.start(); prompt.get([ @@ -128,13 +128,13 @@ function passwd(userID) { ], (err, result) => { if (err) { console.error(err); - mongoose.disconnect(); + util.shutdown(); return; } if (result.password !== result.confirmPassword) { console.error(new Error('Password mismatch')); - mongoose.disconnect(); + util.shutdown(1); return; } @@ -142,11 +142,11 @@ function passwd(userID) { .changePassword(userID, result.password) .then(() => { console.log('Password changed.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); }); } @@ -155,9 +155,6 @@ function passwd(userID) { * Updates the user from the options array. */ function updateUser(userID, options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - const updates = []; if (options.email && typeof options.email === 'string' && options.email.length > 0) { @@ -189,11 +186,11 @@ function updateUser(userID, options) { .all(updates.map((q) => q.exec())) .then(() => { console.log(`User ${userID} updated.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -201,10 +198,6 @@ function updateUser(userID, options) { * Lists all the users registered in the database. */ function listUsers() { - const Table = require('cli-table'); - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .all() .then((users) => { @@ -229,11 +222,11 @@ function listUsers() { }); console.log(table.toString()); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -243,18 +236,15 @@ function listUsers() { * @param {String} srcUserID id of the user to which is the source of the merge */ function mergeUsers(dstUserID, srcUserID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .mergeUsers(dstUserID, srcUserID) .then(() => { console.log(`User ${srcUserID} was merged into user ${dstUserID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -264,18 +254,15 @@ function mergeUsers(dstUserID, srcUserID) { * @param {String} role the role to add */ function addRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .addRoleToUser(userID, role) .then(() => { console.log(`Added the ${role} role to User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -285,18 +272,15 @@ function addRole(userID, role) { * @param {String} role the role to remove */ function removeRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .removeRoleFromUser(userID, role) .then(() => { console.log(`Removed the ${role} role from User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -305,18 +289,15 @@ function removeRole(userID, role) { * @param {String} userID the ID of a user to disable */ function disableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .disableUser(userID) .then(() => { console.log(`User ${userID} was disabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -325,18 +306,15 @@ function disableUser(userID) { * @param {String} userID the ID of a user to enable */ function enableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .enableUser(userID) .then(() => { console.log(`User ${userID} was enabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -408,4 +386,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/www b/bin/www deleted file mode 100755 index 3e9e20918..000000000 --- a/bin/www +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env node - -/** - * Setup the debug paramater. - */ - -process.env.DEBUG = process.env.TALK_DEBUG; - -/** - * Module dependencies. - */ - -const app = require('../app'); -const debug = require('debug')('talk:server'); -const http = require('http'); -const init = require('../init'); -const port = normalizePort(process.env.TALK_PORT || '3000'); - -let server; - -init().then(() => { - - /** - * Get port from environment and store in Express. - */ - app.set('port', port); - - /** - * Create HTTP server. - */ - server = http.createServer(app); - - /** - * Listen on provided port, on all network interfaces. - */ - server.listen(port); - server.on('error', onError); - server.on('listening', onListening); -}); - -/** - * Normalize a port into a number, string, or false. - */ - -function normalizePort(val) { - let port = parseInt(val, 10); - - if (isNaN(port)) { - // named pipe - return val; - } - - if (port >= 0) { - // port number - return port; - } - - return false; -} - -/** - * Event listener for HTTP server "error" event. - */ - -function onError(error) { - if (error.syscall !== 'listen') { - throw error; - } - - let bind = typeof port === 'string' - ? `Pipe ${ port}` - : `Port ${ port}`; - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - console.error(`${bind} requires elevated privileges`); - break; - case 'EADDRINUSE': - console.error(`${bind} is already in use`); - break; - } - - throw error; -} - -/** - * Event listener for HTTP server "listening" event. - */ - -function onListening() { - let addr = server.address(); - let bind = typeof addr === 'string' - ? `pipe ${ addr}` - : `port ${ addr.port}`; - debug(`Listening on ${ bind}`); -} diff --git a/client/coral-embed-stream/src/CommentStream.js b/client/coral-embed-stream/src/CommentStream.js index a0eaca044..40d249088 100644 --- a/client/coral-embed-stream/src/CommentStream.js +++ b/client/coral-embed-stream/src/CommentStream.js @@ -61,8 +61,12 @@ class CommentStream extends Component { // Set up messaging between embedded Iframe an parent component // Using recommended Pym init code which violates .eslint standards const pym = new Pym.Child({polling: 100}); - const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl); - this.props.getStream(path && path[1] || window.location); + + if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) { + this.props.getStream(pym.parentUrl); + } else { + this.props.getStream(window.location); + } } render () { diff --git a/client/coral-framework/actions/auth.js b/client/coral-framework/actions/auth.js index 7313bb30f..1058edbbb 100644 --- a/client/coral-framework/actions/auth.js +++ b/client/coral-framework/actions/auth.js @@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error}); export const checkLogin = () => dispatch => { dispatch(checkLoginRequest()); fetch(`${base}/auth`, getInit('GET')) - .then(handleResp) + .then((res) => { + if (res.status !== 200) { + throw new Error('not logged in'); + } + + return res.json(); + }) .then(user => dispatch(checkLoginSuccess(user))) .catch(error => dispatch(checkLoginFailure(error))); }; diff --git a/models/asset.js b/models/asset.js index 1092cd3e0..b5321cec3 100644 --- a/models/asset.js +++ b/models/asset.js @@ -3,7 +3,6 @@ const uuid = require('uuid'); const Schema = mongoose.Schema; const AssetSchema = new Schema({ - id: { type: String, default: uuid.v4, @@ -19,12 +18,18 @@ const AssetSchema = new Schema({ type: String, default: 'article' }, - headline: String, - summary: String, + scraped: { + type: Date, + default: null + }, + title: String, + description: String, + image: String, section: String, subsection: String, - authors: [String], - publication_date: Date + author: String, + publication_date: Date, + modified_date: Date }, { versionKey: false, timestamps: { @@ -36,80 +41,42 @@ const AssetSchema = new Schema({ /** * Search for assets. Currently only returns all. */ -AssetSchema.statics.search = function(query) { - - return Asset.find(query).exec(); - -}; +AssetSchema.statics.search = (query) => Asset.find(query); /** * Finds an asset by its id. * @param {String} id identifier of the asset (uuid). */ -AssetSchema.statics.findById = function(id) { - - return Asset.findOne({id}).exec(); - -}; +AssetSchema.statics.findById = (id) => Asset.findOne({id}); /** * Finds a asset by its url. * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findByUrl = function(url) { - - return Asset.findOne({'url': url}).exec(); - -}; +AssetSchema.statics.findByUrl = (url) => Asset.findOne({url}); /** * Finds a asset by its url. + * + * NOTE: This function has scalability concerns regarding mongoose's decision + * always write {updated_at: new Date()} on every call to findOneAndUpdate + * even though the update document exactly matches the query document... In + * the future this function should never update, only findOneAndCreate but this + * is not possible with the mongoose driver. + * * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findOrCreateByUrl = function(url) { +AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, { - return Asset.findOne({url}) - .then((asset) => asset ? asset - : Asset.upsert({url})); -}; + // Ensure that if it's new, we return the new object created. + new: true, -/** - * Upserts an asset. -*/ -AssetSchema.statics.upsert = function(data) { - // If an id is not sent, create one. - if (typeof data.id === 'undefined') { - data.id = uuid.v4(); - } + // Perform an upsert in the event that this doesn't exist. + upsert: true, - // Perform the upsert against the id field. - let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec() - .then(() => { - - // Pull the freshly minted asset out and return. - return Asset.findById(data.id); - - }) - .catch((err) => { - - console.error('Error upserting asset.', err); - //return new Promise(); // ??? what do we return on error? - - }); - - return updatePromise; - -}; - -/** - * Remove assets from the db. - * @param {String} query bson query to identify assets to be removed. -*/ -AssetSchema.statics.removeAll = function(query) { - - return Asset.remove(query).exec(); - -}; + // Set the default values if not provided based on the mongoose models. + setDefaultsOnInsert: true +}); const Asset = mongoose.model('Asset', AssetSchema); diff --git a/package.json b/package.json index 9300c347f..bc2f72f20 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,14 @@ "description": "A commenting platform from The Coral Project. https://coralproject.net", "main": "app.js", "scripts": { - "start": "./bin/www", + "start": "./bin/cli serve --jobs", "build": "NODE_ENV=production webpack --config webpack.config.js --bail", "build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch", "lint": "eslint bin/* .", "lint-fix": "eslint . --fix", - "test": "mocha --compilers js:babel-core/register --recursive tests", - "test-watch": "mocha --compilers js:babel-core/register --recursive -w tests", - "embed-start": "NODE_ENV=development npm run build && ./bin/www" + "test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests", + "test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests", + "embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs" }, "config": { "pre-git": { @@ -45,14 +45,16 @@ "express-session": "^1.14.2", "helmet": "^3.1.0", "jsonwebtoken": "^7.1.9", + "kue": "^0.11.5", "lodash": "^4.16.6", + "metascraper": "^1.0.6", "mongoose": "^4.6.5", "morgan": "^1.7.0", + "natural": "^0.4.0", "nodemailer": "^2.6.4", "passport": "^0.3.2", "passport-facebook": "^2.1.1", "passport-local": "^1.0.0", - "natural": "^0.4.0", "prompt": "^1.0.0", "react-linkify": "^0.1.3", "redis": "^2.6.3", diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js index 217c49547..18fd0b7ec 100644 --- a/routes/api/asset/index.js +++ b/routes/api/asset/index.js @@ -1,33 +1,81 @@ const express = require('express'); const router = express.Router(); -const Asset = require('../../../models/asset'); -// Search assets. +const Asset = require('../../../models/asset'); +const scraper = require('../../../services/scraper'); + +// List assets. router.get('/', (req, res, next) => { - let query = {}; + const { + limit = 20, + skip = 0, + sort = 'asc', + field = 'created_at' + } = req.query; - if (typeof req.query.url !== 'undefined') { - query.url = req.query.url; - } + // Find all the assets. + Promise.all([ + Asset + .find({}) + .sort({[field]: (sort === 'asc') ? 1 : -1}) + .skip(skip) + .limit(limit), + Asset.count() + ]) + .then(([result, count]) => { - Asset.search(query) - .then((asset) => { - res.json(asset); - }) - .catch(next); + // Send back the asset data. + res.json({ + result, + count + }); + }) + .catch((err) => { + next(err); + }); }); -// Get an asset by id -router.get('/:id', (req, res, next) => { +// Get an asset by id. +router.get('/:asset_id', (req, res, next) => { - Asset.findById(req.params.id) + // Send back the asset. + Asset + .findById(req.params.asset_id) .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + res.json(asset); }) - .catch(next); + .catch((err) => { + next(err); + }); +}); +// Adds the asset id to the queue to be scraped. +router.post('/:asset_id/scrape', (req, res, next) => { + + // Create a new asset scrape job. + Asset + .findById(req.params.asset_id) + .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + + return scraper.create(asset); + }) + .then((job) => { + + // Send the job back for monitoring. + res.status(201).json(job); + }) + .catch((err) => { + next(err); + }); }); module.exports = router; diff --git a/routes/api/auth/index.js b/routes/api/auth/index.js index b88b6fe12..369e5ec77 100644 --- a/routes/api/auth/index.js +++ b/routes/api/auth/index.js @@ -7,7 +7,18 @@ const router = express.Router(); /** * This returns the user if they are logged in. */ -router.get('/', authorization.needed(), (req, res) => { +router.get('/', (req, res, next) => { + if (req.user) { + return next(); + } + + // When there is no user on the request, then just send back a 204 to this + // request. It's not really "an error" if what they asked for isn't available, + // but it could be. + res.status(204).end(); +}, (req, res) => { + + // Send back the user object. res.json(req.user.toObject()); }); diff --git a/routes/api/index.js b/routes/api/index.js index 8c6014ed8..8da3f791b 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -3,13 +3,18 @@ const authorization = require('../../middleware/authorization'); const router = express.Router(); -router.use('/asset', require('./asset')); -router.use('/auth', require('./auth')); -router.use('/comments', authorization.needed(), require('./comments')); -router.use('/queue', require('./queue')); +router.use('/asset', authorization.needed('admin'), require('./asset')); router.use('/settings', authorization.needed('admin'), require('./settings')); -router.use('/stream', require('./stream')); -router.use('/user', require('./user')); +router.use('/queue', authorization.needed('admin'), require('./queue')); + +router.use('/comments', authorization.needed(), require('./comments')); router.use('/actions', authorization.needed(), require('./actions')); +router.use('/auth', require('./auth')); +router.use('/stream', require('./stream')); +router.use('/user', require('./user')); + +// Bind the kue handler to the /kue path. +router.use('/kue', authorization.needed('admin'), require('kue').app); + module.exports = router; diff --git a/routes/api/queue/index.js b/routes/api/queue/index.js index edc5602ef..f661992f1 100644 --- a/routes/api/queue/index.js +++ b/routes/api/queue/index.js @@ -1,7 +1,6 @@ const express = require('express'); const Comment = require('../../../models/comment'); const Setting = require('../../../models/setting'); -const authorization = require('../../../middleware/authorization'); const router = express.Router(); @@ -13,7 +12,7 @@ const router = express.Router(); // depending on the settings. The :moderation overwrites this settings. // Pre-moderation: New comments are shown in the moderator queues immediately. // Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users. -router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => { +router.get('/comments/pending', (req, res, next) => { Setting.getModerationSetting().then(function({moderation}){ Comment.moderationQueue(moderation).then((comments) => { res.status(200).json(comments); diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index a48548608..0ef97aad2 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -1,52 +1,99 @@ const express = require('express'); const _ = require('lodash'); +const scraper = require('../../../services/scraper'); const Comment = require('../../../models/comment'); const User = require('../../../models/user'); const Action = require('../../../models/action'); const Asset = require('../../../models/asset'); - const Setting = require('../../../models/setting'); const router = express.Router(); -// Find all the comments by a specific asset_url. -// . if pre: get the comments that are accepted. -// . if post: get the comments that are new and accepted. router.get('/', (req, res, next) => { // Get the asset_id for this url (or create it if it doesn't exist) Promise.all([ - Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)), + + // Find or create the asset by url. + Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)) + + // Add the found asset to the scraper if it's not already scraped. + .then((asset) => { + if (!asset.scraped) { + return scraper.create(asset).then(() => asset); + } + + return asset; + }), + + // Get the moderation setting from the settings. Setting.getModerationSetting() ]) .then(([asset, {moderation}]) => { - // Get the sitewide moderation setting and return the appropriate comments - switch(moderation){ - case 'pre': - return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]); - case 'post': - return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]); - default: - return Promise.reject(new Error('Moderation setting not found.')); + let comments; + + if (moderation === 'post') { + comments = Comment.findAcceptedByAssetId(asset.id); + } else { + + // Defaults to 'pre' moderation. + comments = Comment.findAcceptedAndNewByAssetId(asset.id); } + + return Promise.all([ + + // This is the promised component... Fetch the comments based on the + // moderation settings. + comments, + + // Send back the reference to the asset. + asset + ]); }) // Get all the users and actions for those comments. .then(([comments, asset]) => { + + // Get the user id's from the author id's as a unique array that gets + // sorted. + let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort(); + + // Fetch the users for which there is a comment available for them. + let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : []; + + // Fetch the actions for pretty much everything at this point. + let actions = Action.getActionSummaries(_.uniq([ + + // Actions can be on assets... + asset.id, + + // Comments... + ...comments.map((comment) => comment.id), + + // Or Authors... + ...userIDs + ]), req.user ? req.user.id : false); + return Promise.all([ - [asset], + + // Pass back the asset that we loaded... + asset, + + // It's comments... comments, - User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))), - Action.getActionSummaries(_.uniq([ - asset.id, - ...comments.map((comment) => comment.id), - ...comments.map((comment) => comment.author_id) - ]), req.user ? req.user.id : '') + + // All the users/authors of those comments... + users, + + // And all actions about the asset, comments, and users. + actions ]); }) - .then(([assets, comments, users, actions]) => { + .then(([asset, comments, users, actions]) => { + + // Send back the payload containing all this data. res.json({ - assets, + assets: [asset], comments, users, actions diff --git a/routes/api/user/index.js b/routes/api/user/index.js index 3251e0608..43d3b8ac0 100644 --- a/routes/api/user/index.js +++ b/routes/api/user/index.js @@ -16,7 +16,7 @@ router.get('/', authorization.needed('admin'), (req, res, next) => { page = 1, asc = 'false', limit = 50 // Total Per Page - } = req.query; + } = req.query; Promise.all([ User @@ -128,9 +128,10 @@ router.post('/request-password-reset', (req, res, next) => { return mailer.sendSimple(options); }) .then(() => { + // we want to send a 204 regardless of the user being found in the db // if we fail on missing emails, it would reveal if people are registered or not. - res.status(204).send('OK'); + res.status(204).end(); }) .catch(error => { const errorMsg = typeof error === 'string' ? error : error.message; diff --git a/routes/index.js b/routes/index.js index 0c03edcc5..a875d5127 100644 --- a/routes/index.js +++ b/routes/index.js @@ -6,11 +6,11 @@ router.use('/admin', require('./admin')); router.use('/embed', require('./embed')); router.get('/', (req, res) => { - return res.render('article', {title: 'Coral Talk'}); + res.render('article', {title: 'Coral Talk'}); }); router.get('/assets/:asset_title', (req, res) => { - return res.render('article', {title: req.params.asset_title.split('-').join(' ')}); + res.render('article', {title: req.params.asset_title.split('-').join(' ')}); }); module.exports = router; diff --git a/services/scraper.js b/services/scraper.js new file mode 100644 index 000000000..665d4fc66 --- /dev/null +++ b/services/scraper.js @@ -0,0 +1,140 @@ +const kue = require('kue'); +const queue = kue.createQueue(); +const debug = require('debug')('talk:services:scraper'); +const Asset = require('../models/asset'); +const JOB_NAME = 'scraper'; + +const metascraper = require('metascraper'); + +/** + * Exposes a service object to allow operations to execute against the scraper. + * @type {Object} + */ +const scraper = { + + /** + * creates a new scraper job and scrapes the url when it gets processed. + */ + create(asset) { + return new Promise((resolve, reject) => { + debug(`Creating job for Asset[${asset.id}]`); + + let job = queue + .create(JOB_NAME, { + title: `Scrape for asset ${asset.id}`, + asset_id: asset.id + }) + .attempts(10) + .delay(1000) + .backoff({type: 'exponential'}) + .save((err) => { + if (err) { + return reject(err); + } + + debug(`Created Job[${job.id}] for Asset[${asset.id}]`); + + return resolve(job); + }); + }); + }, + + /** + * Scrapes the given asset for metadata. + */ + scrape(asset) { + return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, { + section: ($) => $('meta[property="article:section"]').attr('content'), + modified: ($) => $('meta[property="article:modified"]').attr('content') + })); + }, + + update(id, meta) { + return Asset.update({id}, { + $set: { + title: meta.title || '', + description: meta.description || '', + image: meta.image ? meta.image : '', + author: meta.author || '', + publication_date: meta.date || '', + modified_date: meta.modified || '', + section: meta.section || '', + scraped: new Date() + } + }); + }, + + /** + * Start the queue processor for the scraper job. + */ + process() { + + debug(`Now processing ${JOB_NAME} jobs`); + + // Process jobs with the processJob function. + queue.process(JOB_NAME, (job, done) => { + + debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + Asset + + // Find the asset, or complain that it doesn't exist. + .findById(job.data.asset_id) + .then((asset) => { + if (!asset) { + throw new Error('asset not found'); + } + + return asset; + }) + + // Scrape the metadata from the asset. + .then(scraper.scrape) + + // Assign the metadata retrieved for the asset to the db. + .then((meta) => { + debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + return scraper.update(job.data.asset_id, meta); + }) + + // Finish the job because we just handled our scraping + updating the + // asset in the database. + .then(() => { + debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`); + done(); + }) + + // Handle errors that occur. + .catch((err) => { + console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err); + + done(err); + }); + }); + }, + + /** + * Shuts down the current queue to ensure that the application can shutdown + * cleanly. + */ + shutdown() { + return new Promise((resolve, reject) => { + + // Shutdown and give the queue 5 seconds to shutdown before we start + // killing jobs. + queue.shutdown(5000, (err) => { + if (err) { + return reject(err); + } + + debug(`Processing for ${JOB_NAME} jobs stopped`); + + resolve(); + }); + }); + } + +}; + +module.exports = scraper; diff --git a/swagger.yaml b/swagger.yaml index fd3b0012b..09adb97d6 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -256,6 +256,86 @@ paths: description: An error occured. schema: $ref: '#/definitions/Error' + /asset: + get: + parameters: + - name: limit + in: query + type: number + format: int32 + description: Limit the listing results + - name: skip + in: query + type: number + format: int32 + description: Skip the listing results + - name: sort + in: query + enum: + - asc + - desc + type: string + description: Sorting method + - name: field + in: query + type: string + description: Field to sort by. + responses: + 200: + description: Assets listed. + schema: + type: object + properties: + count: + type: number + description: Total number of assets found. + result: + type: array + items: + $ref: '#/definitions/Asset' + + /asset/{asset_id}: + get: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 200: + description: The requested asset. + schema: + $ref: '#/definitions/Asset' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + + /asset/{asset_id}/scrape: + post: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 201: + description: The job that was created. + schema: + $ref: '#/definitions/Job' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + /stream: get: tags: @@ -420,3 +500,10 @@ definitions: type: object Settings: type: object + Job: + type: object + properties: + id: + format: number + state: + format: string diff --git a/tests/index.js b/tests/index.js deleted file mode 100644 index 0c3d947ce..000000000 --- a/tests/index.js +++ /dev/null @@ -1,9 +0,0 @@ -const expect = require('chai').expect; - -describe('Comment', () => { - describe('#add', () => { - it('should add a comment', () => { - expect(0).to.be.equal(0); - }); - }); -}); diff --git a/tests/models/action.js b/tests/models/action.js index 87d53b7da..c354b8f83 100644 --- a/tests/models/action.js +++ b/tests/models/action.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Action = require('../../models/action'); const expect = require('chai').expect; diff --git a/tests/models/asset.js b/tests/models/asset.js index d0c3d3429..a5c4ae62b 100644 --- a/tests/models/asset.js +++ b/tests/models/asset.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Asset = require('../../models/asset'); const expect = require('chai').expect; @@ -74,35 +70,4 @@ describe('Asset: model', () => { }); }); }); - - describe('#upsert', ()=> { - it('should insert an asset with no id', () => { - return Asset.upsert({url: 'http://newasset.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id'); - }); - }); - - it('should update an asset when the id exists', () => { - return Asset.upsert({id: 1, url: 'http://new.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id') - .and.to.equal('1'); - expect(asset).to.have.property('url') - .and.to.equal('http://new.test.com'); - }); - }); - }); - - describe('#removeAll', ()=> { - it('should insert an asset with no id', () => { - return Asset.removeAll({id:1}) - .then(() => { - return Asset.findById(1); - }) - .then((result) => { - expect(result).to.be.null; - }); - }); - }); }); diff --git a/tests/models/comment.js b/tests/models/comment.js index f8dd7f9f5..07834a186 100644 --- a/tests/models/comment.js +++ b/tests/models/comment.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Comment = require('../../models/comment'); const User = require('../../models/user'); const Action = require('../../models/action'); diff --git a/tests/models/setting.js b/tests/models/setting.js index 3f4dc27fa..a36b363fb 100644 --- a/tests/models/setting.js +++ b/tests/models/setting.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Setting = require('../../models/setting'); const expect = require('chai').expect; diff --git a/tests/models/user.js b/tests/models/user.js index f77543b4b..c8af92054 100644 --- a/tests/models/user.js +++ b/tests/models/user.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const User = require('../../models/user'); const expect = require('chai').expect; diff --git a/tests/utils/mongoose.js b/tests/mongoose.js similarity index 71% rename from tests/utils/mongoose.js rename to tests/mongoose.js index 1549440a6..9d3ba1195 100644 --- a/tests/utils/mongoose.js +++ b/tests/mongoose.js @@ -1,8 +1,4 @@ -const mongoose = require('../../mongoose'); - -// Ensure the NODE_ENV is set to 'test', -// this is helpful when you would like to change behavior when testing. -process.env.NODE_ENV = 'test'; +const mongoose = require('../mongoose'); beforeEach(function (done) { function clearDB() { diff --git a/tests/utils/passport.js b/tests/passport.js similarity index 90% rename from tests/utils/passport.js rename to tests/passport.js index 401e50b77..2d1c53ab9 100644 --- a/tests/utils/passport.js +++ b/tests/passport.js @@ -1,4 +1,4 @@ -const authorization = require('../../middleware/authorization'); +const authorization = require('../middleware/authorization'); // Add the passport middleware here before it's setup. authorization.middleware.push((req, res, next) => { diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js index 542e88c42..fb67b7b48 100644 --- a/tests/routes/api/assets/index.js +++ b/tests/routes/api/assets/index.js @@ -1,36 +1,10 @@ -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +describe('/assets', () => { -const chai = require('chai'); -const server = require('../../../../app'); + describe('GET', () => { -// Setup chai. -chai.should(); -chai.use(require('chai-http')); + it('should return assets that we search for'); + it('should not return assets that we do not search for'); -describe('Asset: routes', () => { - - describe('/GET Asset', () => { - describe('#get', () => { - it('It should get an empty array when there are no assets.', (done) => { - - chai.request(server) - .get('/api/v1/asset') - .set(passport.inject({roles: ['admin']})) - .end((err, res) => { - - if (err) { - throw new Error(err); - } - - res.should.have.status(200); - res.body.should.be.a('array'); - res.body.length.should.be.eql(0); - done(); - }); - - }); - }); }); }); diff --git a/tests/routes/api/auth/index.js b/tests/routes/api/auth/index.js index dd49a1472..ddca63fe8 100644 --- a/tests/routes/api/auth/index.js +++ b/tests/routes/api/auth/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index 29775d3d0..ff7f3a9d4 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/queue/index.js b/tests/routes/api/queue/index.js index 4eff3d384..74137a49a 100644 --- a/tests/routes/api/queue/index.js +++ b/tests/routes/api/queue/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/settings/index.js b/tests/routes/api/settings/index.js index 86feca1eb..9f4466a7f 100644 --- a/tests/routes/api/settings/index.js +++ b/tests/routes/api/settings/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/stream/index.js b/tests/routes/api/stream/index.js index 009184b6b..7d05c9708 100644 --- a/tests/routes/api/stream/index.js +++ b/tests/routes/api/stream/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; @@ -92,8 +90,8 @@ describe('api/stream: routes', () => { .then(res => { expect(res).to.have.status(200); expect(res.body.assets.length).to.equal(1); - expect(res.body.comments.length).to.equal(1); - expect(res.body.users.length).to.equal(1); + expect(res.body.comments.length).to.equal(2); + expect(res.body.users.length).to.equal(2); expect(res.body.actions.length).to.equal(1); }); }); diff --git a/tests/services/scraper.js b/tests/services/scraper.js new file mode 100644 index 000000000..f3b5d006c --- /dev/null +++ b/tests/services/scraper.js @@ -0,0 +1,22 @@ +describe('scraper: services', () => { + describe('#create', () => { + it('should create a new kue job'); + }); + + describe('#scrape', () => { + it('should scrape complete information'); + it('should scrape what it can'); + }); + + describe('#update', () => { + it('should update the database record entries from the meta'); + }); + + describe('#process', () => { + it('should start the processor to scrape assets'); + }); + + describe('#shutdown', () => { + it('should shutdown the job processor'); + }); +}); diff --git a/util.js b/util.js new file mode 100644 index 000000000..6907d90cd --- /dev/null +++ b/util.js @@ -0,0 +1,42 @@ +const util = module.exports = {}; + +/** + * Stores an array of functions that should be executed in the event that the + * application needs to shutdown. + * @type {Array} + */ +util.toshutdown = []; + +/** + * Calls all the shutdown functions and then ends the process. + * @param {Number} [defaultCode=0] default return code upon sucesfull shutdown. + */ +util.shutdown = (defaultCode = 0) => { + Promise + .all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func)) + .then(() => { + process.exit(defaultCode); + }) + .catch((err) => { + console.error(err); + + process.exit(1); + }); +}; + +/** + * Waits until an event is triggered by the node runtime and elevates a series + * of jobs to be ran in the event we need to shutdown. + * @param {Array} jobs Array of promise capable shutdown functions that are + * executed. + */ +util.onshutdown = (jobs) => { + + // Add the new jobs to shutdown to the object reference. + util.toshutdown = util.toshutdown.concat(jobs); +}; + +// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the +// event that we have an external event. +process.on('SIGTERM', () => util.shutdown()); +process.on('SIGINT', () => util.shutdown()); diff --git a/views/article.ejs b/views/article.ejs index 7d168f960..71db59593 100644 --- a/views/article.ejs +++ b/views/article.ejs @@ -1,8 +1,10 @@ - + + +