diff --git a/.eslintrc.json b/.eslintrc.json
index 6ac5a08e6..035a86189 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -5,25 +5,15 @@
},
"extends": "eslint:recommended",
"rules": {
- "indent": [
- "error",
+ "indent": ["error",
2
],
"no-console": [
0
],
- "linebreak-style": [
- "error",
- "unix"
- ],
- "quotes": [
- "error",
- "single"
- ],
- "semi": [
- "error",
- "always"
- ],
+ "linebreak-style": ["error", "unix"],
+ "quotes": ["error", "single"],
+ "semi": ["error", "always"],
"no-template-curly-in-string": [1],
"no-unsafe-negation": [1],
"array-callback-return": [1],
@@ -35,7 +25,6 @@
"no-throw-literal": [2],
"yoda": [1],
"no-path-concat": [2],
- "no-process-exit": [2],
"eol-last": [1],
"no-continue": [1],
"no-nested-ternary": [1],
@@ -46,20 +35,20 @@
"no-const-assign": [2],
"no-duplicate-imports": [2],
"prefer-template": [1],
- "comma-spacing": [
- "error",
- {
+ "comma-spacing": ["error", {
"after": true
- }
- ],
+ }],
"no-var": [2],
"no-lonely-if": [2],
"curly": [2],
- "no-unused-vars": ["error", { "argsIgnorePattern": "next" }],
- "no-multiple-empty-lines": [
- "error",
- {"max": 1}
- ],
- "newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }]
+ "no-unused-vars": ["error", {
+ "argsIgnorePattern": "next"
+ }],
+ "no-multiple-empty-lines": ["error", {
+ "max": 1
+ }],
+ "newline-per-chained-call": ["error", {
+ "ignoreChainWithDepth": 2
+ }]
}
}
diff --git a/bin/cli b/bin/cli
index bec5ea84f..d8148282f 100755
--- a/bin/cli
+++ b/bin/cli
@@ -19,7 +19,10 @@ const pkg = require('../package.json');
program
.version(pkg.version)
+ .command('serve', 'serve the application')
+ .command('assets', 'interact with assets')
.command('settings', 'work with the application settings')
+ .command('jobs', 'work with the job queues')
.command('users', 'work with the application auth')
.parse(process.argv);
diff --git a/bin/cli-assets b/bin/cli-assets
new file mode 100755
index 000000000..99965c6d9
--- /dev/null
+++ b/bin/cli-assets
@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+
+/**
+ * Setup the debug paramater.
+ */
+
+process.env.DEBUG = process.env.TALK_DEBUG;
+
+/**
+ * Module dependencies.
+ */
+
+const program = require('commander');
+const pkg = require('../package.json');
+const Table = require('cli-table');
+const Asset = require('../models/asset');
+const mongoose = require('../mongoose');
+const util = require('../util');
+
+// Register the shutdown criteria.
+util.onshutdown([
+ () => mongoose.disconnect()
+]);
+
+/**
+ * Lists all the assets registered in the database.
+ */
+function listAssets() {
+ Asset
+ .find({})
+ .sort({'created_at': 1})
+ .then((asset) => {
+ let table = new Table({
+ head: [
+ 'ID',
+ 'Title',
+ 'URL'
+ ]
+ });
+
+ asset.forEach((asset) => {
+ table.push([
+ asset.id,
+ asset.title ? asset.title : '',
+ asset.url ? asset.url : ''
+ ]);
+ });
+
+ console.log(table.toString());
+ util.shutdown();
+ })
+ .catch((err) => {
+ console.error(err);
+ util.shutdown(1);
+ });
+}
+
+//==============================================================================
+// Setting up the program command line arguments.
+//==============================================================================
+
+program
+ .version(pkg.version);
+
+program
+ .command('list')
+ .description('list all the assets in the database')
+ .action(listAssets);
+
+program.parse(process.argv);
+
+// If there is no command listed, output help.
+if (!process.argv.slice(2).length) {
+ program.outputHelp();
+ util.shutdown();
+}
diff --git a/bin/cli-jobs b/bin/cli-jobs
new file mode 100755
index 000000000..f14276d38
--- /dev/null
+++ b/bin/cli-jobs
@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+/**
+ * Setup the debug paramater.
+ */
+
+process.env.DEBUG = process.env.TALK_DEBUG;
+
+/**
+ * Module dependencies.
+ */
+
+const program = require('commander');
+const scraper = require('../services/scraper');
+const util = require('../util');
+const mongoose = require('../mongoose');
+
+util.onshutdown([
+ () => mongoose.disconnect()
+]);
+
+function processJobs() {
+
+ // Start the processor.
+ scraper.process();
+
+ // The scraper only needs to shutdown when the scraper has actually been
+ // started.
+ util.onshutdown([
+ () => scraper.shutdown()
+ ]);
+}
+
+//==============================================================================
+// Setting up the program command line arguments.
+//==============================================================================
+
+program
+ .command('process')
+ .description('starts job processing')
+ .action(processJobs);
+
+program.parse(process.argv);
+
+// If there is no command listed, output help.
+if (process.argv.length <= 2) {
+ program.outputHelp();
+ util.shutdown();
+}
diff --git a/bin/cli-serve b/bin/cli-serve
new file mode 100755
index 000000000..14c1261d7
--- /dev/null
+++ b/bin/cli-serve
@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+
+/**
+ * Setup the debug paramater.
+ */
+
+process.env.DEBUG = process.env.TALK_DEBUG;
+
+const app = require('../app');
+const debug = require('debug')('talk:server');
+const http = require('http');
+const init = require('../init');
+const scraper = require('../services/scraper');
+const mongoose = require('../mongoose');
+const util = require('../util');
+
+/**
+* Get port from environment and store in Express.
+*/
+const port = normalizePort(process.env.TALK_PORT || '3000');
+
+app.set('port', port);
+
+/**
+* Create HTTP server.
+*/
+const server = http.createServer(app);
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+function onError(error) {
+ if (error.syscall !== 'listen') {
+ throw error;
+ }
+
+ let bind = typeof port === 'string'
+ ? `Pipe ${port}`
+ : `Port ${port}`;
+
+ // handle specific listen errors with friendly messages
+ switch (error.code) {
+ case 'EACCES':
+ console.error(`${bind} requires elevated privileges`);
+ break;
+ case 'EADDRINUSE':
+ console.error(`${bind} is already in use`);
+ break;
+ }
+
+ throw error;
+}
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+ let port = parseInt(val, 10);
+
+ if (isNaN(port)) {
+ // named pipe
+ return val;
+ }
+
+ if (port >= 0) {
+ // port number
+ return port;
+ }
+
+ return false;
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+ let addr = server.address();
+ let bind = typeof addr === 'string'
+ ? `pipe ${ addr}`
+ : `port ${ addr.port}`;
+ debug(`Listening on ${ bind}`);
+}
+
+/**
+ * Start the app.
+ */
+function startApp() {
+ init().then(() => {
+
+ /**
+ * Listen on provided port, on all network interfaces.
+ */
+ server.listen(port);
+ server.on('error', onError);
+ server.on('listening', onListening);
+ });
+}
+
+/**
+ * Module dependencies.
+ */
+
+const program = require('commander');
+
+//==============================================================================
+// Setting up the program command line arguments.
+//==============================================================================
+
+program
+ .option('-j, --jobs', 'enable job processing on this thread')
+ .parse(process.argv);
+
+// Start the application serving.
+startApp();
+
+// Enable job processing on the thread if enabled.
+if (program.jobs) {
+
+ // Start the processor.
+ scraper.process();
+}
+
+// Define a safe shutdown function to call in the event we need to shutdown
+// because the node hooks are below which will interrupt the shutdown process.
+// Shutdown the mongoose connection, the app server, and the scraper.
+util.onshutdown([
+ () => program.jobs ? scraper.shutdown() : null,
+ () => mongoose.disconnect(),
+ () => server.close()
+]);
diff --git a/bin/cli-settings b/bin/cli-settings
index cff6c04ed..c639a5ca5 100755
--- a/bin/cli-settings
+++ b/bin/cli-settings
@@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG;
*/
const program = require('commander');
+const mongoose = require('../mongoose');
+const Setting = require('../models/setting');
+const util = require('../util');
+
+// Regeister the shutdown criteria.
+util.onshutdown([
+ () => mongoose.disconnect()
+]);
//==============================================================================
// Setting up the program command line arguments.
@@ -20,19 +28,17 @@ program
.command('init')
.description('initilizes the talk settings')
.action(() => {
- const mongoose = require('../mongoose');
- const Setting = require('../models/setting');
const defaults = {id: '1', moderation: 'pre'};
Setting
.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true})
.then(() => {
console.log('Created settings object.');
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(`failed to create the settings object ${JSON.stringify(err)}`);
- throw new Error(err); // just to be safe
+ util.shutdown(1);
});
});
@@ -41,4 +47,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
+ util.shutdown();
}
diff --git a/bin/cli-users b/bin/cli-users
index 7f5e56d80..723d84074 100755
--- a/bin/cli-users
+++ b/bin/cli-users
@@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG;
const program = require('commander');
const pkg = require('../package.json');
const prompt = require('prompt');
+const User = require('../models/user');
+const mongoose = require('../mongoose');
+const util = require('../util');
+const Table = require('cli-table');
+
+// Regeister the shutdown criteria.
+util.onshutdown([
+ () => mongoose.disconnect()
+]);
/**
* Prompts for input and registers a user based on those.
*/
function createUser(options) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
return new Promise((resolve, reject) => {
if (options.flag_mode) {
@@ -74,11 +80,11 @@ function createUser(options) {
})
.then((user) => {
console.log(`Created user ${user.id}.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown();
});
}
@@ -86,20 +92,17 @@ function createUser(options) {
* Deletes a user.
*/
function deleteUser(userID) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.findOneAndRemove({
id: userID
})
.then(() => {
console.log('Deleted user');
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown();
});
}
@@ -107,9 +110,6 @@ function deleteUser(userID) {
* Changes the password for a user.
*/
function passwd(userID) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
prompt.start();
prompt.get([
@@ -128,13 +128,13 @@ function passwd(userID) {
], (err, result) => {
if (err) {
console.error(err);
- mongoose.disconnect();
+ util.shutdown();
return;
}
if (result.password !== result.confirmPassword) {
console.error(new Error('Password mismatch'));
- mongoose.disconnect();
+ util.shutdown(1);
return;
}
@@ -142,11 +142,11 @@ function passwd(userID) {
.changePassword(userID, result.password)
.then(() => {
console.log('Password changed.');
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
});
}
@@ -155,9 +155,6 @@ function passwd(userID) {
* Updates the user from the options array.
*/
function updateUser(userID, options) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
const updates = [];
if (options.email && typeof options.email === 'string' && options.email.length > 0) {
@@ -189,11 +186,11 @@ function updateUser(userID, options) {
.all(updates.map((q) => q.exec()))
.then(() => {
console.log(`User ${userID} updated.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -201,10 +198,6 @@ function updateUser(userID, options) {
* Lists all the users registered in the database.
*/
function listUsers() {
- const Table = require('cli-table');
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.all()
.then((users) => {
@@ -231,11 +224,11 @@ function listUsers() {
});
console.log(table.toString());
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -245,18 +238,15 @@ function listUsers() {
* @param {String} srcUserID id of the user to which is the source of the merge
*/
function mergeUsers(dstUserID, srcUserID) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.mergeUsers(dstUserID, srcUserID)
.then(() => {
console.log(`User ${srcUserID} was merged into user ${dstUserID}.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -266,18 +256,15 @@ function mergeUsers(dstUserID, srcUserID) {
* @param {String} role the role to add
*/
function addRole(userID, role) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.addRoleToUser(userID, role)
.then(() => {
console.log(`Added the ${role} role to User ${userID}.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -287,18 +274,15 @@ function addRole(userID, role) {
* @param {String} role the role to remove
*/
function removeRole(userID, role) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.removeRoleFromUser(userID, role)
.then(() => {
console.log(`Removed the ${role} role from User ${userID}.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -307,18 +291,15 @@ function removeRole(userID, role) {
* @param {String} userID the ID of a user to disable
*/
function disableUser(userID) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.disableUser(userID)
.then(() => {
console.log(`User ${userID} was disabled.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -327,18 +308,15 @@ function disableUser(userID) {
* @param {String} userID the ID of a user to enable
*/
function enableUser(userID) {
- const User = require('../models/user');
- const mongoose = require('../mongoose');
-
User
.enableUser(userID)
.then(() => {
console.log(`User ${userID} was enabled.`);
- mongoose.disconnect();
+ util.shutdown();
})
.catch((err) => {
console.error(err);
- mongoose.disconnect();
+ util.shutdown(1);
});
}
@@ -410,4 +388,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
+ util.shutdown();
}
diff --git a/bin/www b/bin/www
deleted file mode 100755
index 3e9e20918..000000000
--- a/bin/www
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Setup the debug paramater.
- */
-
-process.env.DEBUG = process.env.TALK_DEBUG;
-
-/**
- * Module dependencies.
- */
-
-const app = require('../app');
-const debug = require('debug')('talk:server');
-const http = require('http');
-const init = require('../init');
-const port = normalizePort(process.env.TALK_PORT || '3000');
-
-let server;
-
-init().then(() => {
-
- /**
- * Get port from environment and store in Express.
- */
- app.set('port', port);
-
- /**
- * Create HTTP server.
- */
- server = http.createServer(app);
-
- /**
- * Listen on provided port, on all network interfaces.
- */
- server.listen(port);
- server.on('error', onError);
- server.on('listening', onListening);
-});
-
-/**
- * Normalize a port into a number, string, or false.
- */
-
-function normalizePort(val) {
- let port = parseInt(val, 10);
-
- if (isNaN(port)) {
- // named pipe
- return val;
- }
-
- if (port >= 0) {
- // port number
- return port;
- }
-
- return false;
-}
-
-/**
- * Event listener for HTTP server "error" event.
- */
-
-function onError(error) {
- if (error.syscall !== 'listen') {
- throw error;
- }
-
- let bind = typeof port === 'string'
- ? `Pipe ${ port}`
- : `Port ${ port}`;
-
- // handle specific listen errors with friendly messages
- switch (error.code) {
- case 'EACCES':
- console.error(`${bind} requires elevated privileges`);
- break;
- case 'EADDRINUSE':
- console.error(`${bind} is already in use`);
- break;
- }
-
- throw error;
-}
-
-/**
- * Event listener for HTTP server "listening" event.
- */
-
-function onListening() {
- let addr = server.address();
- let bind = typeof addr === 'string'
- ? `pipe ${ addr}`
- : `port ${ addr.port}`;
- debug(`Listening on ${ bind}`);
-}
diff --git a/client/coral-embed-stream/src/CommentStream.js b/client/coral-embed-stream/src/CommentStream.js
index a0eaca044..40d249088 100644
--- a/client/coral-embed-stream/src/CommentStream.js
+++ b/client/coral-embed-stream/src/CommentStream.js
@@ -61,8 +61,12 @@ class CommentStream extends Component {
// Set up messaging between embedded Iframe an parent component
// Using recommended Pym init code which violates .eslint standards
const pym = new Pym.Child({polling: 100});
- const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl);
- this.props.getStream(path && path[1] || window.location);
+
+ if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) {
+ this.props.getStream(pym.parentUrl);
+ } else {
+ this.props.getStream(window.location);
+ }
}
render () {
diff --git a/client/coral-framework/actions/auth.js b/client/coral-framework/actions/auth.js
index 7313bb30f..1058edbbb 100644
--- a/client/coral-framework/actions/auth.js
+++ b/client/coral-framework/actions/auth.js
@@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error});
export const checkLogin = () => dispatch => {
dispatch(checkLoginRequest());
fetch(`${base}/auth`, getInit('GET'))
- .then(handleResp)
+ .then((res) => {
+ if (res.status !== 200) {
+ throw new Error('not logged in');
+ }
+
+ return res.json();
+ })
.then(user => dispatch(checkLoginSuccess(user)))
.catch(error => dispatch(checkLoginFailure(error)));
};
diff --git a/middleware/authorization.js b/middleware/authorization.js
index 238219a06..65f77b753 100644
--- a/middleware/authorization.js
+++ b/middleware/authorization.js
@@ -2,7 +2,9 @@
* authorization contains the references to the authorization middleware.
* @type {Object}
*/
-const authorization = module.exports = {};
+const authorization = module.exports = {
+ middleware: []
+};
const debug = require('debug')('talk:middleware:authorization');
@@ -33,21 +35,29 @@ authorization.has = (user, ...roles) => roles.every((role) => user.roles.indexOf
* @param {Array} roles all the roles that a user must have
* @return {Callback} connect middleware
*/
-authorization.needed = (...roles) => (req, res, next) => {
- // All routes that are wrapepd with this middleware actually require a role.
- if (!req.user) {
- debug(`No user on request, returning with ${ErrNotAuthorized}`);
- return next(ErrNotAuthorized);
- }
+authorization.needed = (...roles) => [
- // Check to see if the current user has all the roles requested for the given
- // array of roles requested, if one is not on the user, then this will
- // evaluate to true.
- if (!authorization.has(req.user, ...roles)) {
- debug('User does not have all the required roles to access this page');
- return next(ErrNotAuthorized);
- }
+ // Insert the pre-needed middlware.
+ ...authorization.middleware,
- // Looks like they're allowed!
- return next();
-};
+ // Insert the actual middleware to check for the required role.
+ (req, res, next) => {
+
+ // All routes that are wrapepd with this middleware actually require a role.
+ if (!req.user) {
+ debug(`No user on request, returning with ${ErrNotAuthorized}`);
+ return next(ErrNotAuthorized);
+ }
+
+ // Check to see if the current user has all the roles requested for the given
+ // array of roles requested, if one is not on the user, then this will
+ // evaluate to true.
+ if (!authorization.has(req.user, ...roles)) {
+ debug('User does not have all the required roles to access this page');
+ return next(ErrNotAuthorized);
+ }
+
+ // Looks like they're allowed!
+ return next();
+ }
+];
diff --git a/models/action.js b/models/action.js
index 91378b0b0..891dbd4f5 100644
--- a/models/action.js
+++ b/models/action.js
@@ -27,6 +27,35 @@ ActionSchema.statics.findById = function(id) {
return Action.findOne({id});
};
+/**
+ * Add an action.
+ * @param {String} item_id identifier of the comment (uuid)
+ * @param {String} user_id user id of the action (uuid)
+ * @param {String} action the new action to the comment
+ * @return {Promise}
+ */
+ActionSchema.statics.insertUserAction = ({item_id, item_type, user_id, action_type}) => {
+ const action = {
+ item_id,
+ item_type,
+ user_id,
+ action_type
+ };
+
+ // Create/Update the action.
+ return Action.findOneAndUpdate(action, action, {
+
+ // Ensure that if it's new, we return the new object created.
+ new: true,
+
+ // Perform an upsert in the event that this doesn't exist.
+ upsert: true,
+
+ // Set the default values if not provided based on the mongoose models.
+ setDefaultsOnInsert: true
+ });
+};
+
/**
* Finds actions in an array of ids.
* @param {String} ids array of user identifiers (uuid)
@@ -41,7 +70,7 @@ ActionSchema.statics.findByItemIdArray = function(item_ids) {
* Returns summaries of actions for an array of ids
* @param {String} ids array of user identifiers (uuid)
*/
-ActionSchema.statics.getActionSummaries = function(item_ids) {
+ActionSchema.statics.getActionSummaries = function(item_ids, current_user_id = '') {
return Action.aggregate([
{
@@ -71,6 +100,18 @@ ActionSchema.statics.getActionSummaries = function(item_ids) {
// just grabbing the last instance of the item type here.
item_type: {
$last: '$item_type'
+ },
+
+ current_user: {
+ $max: {
+ $cond: {
+ if: {
+ $eq: ['$user_id', current_user_id],
+ },
+ then: '$$CURRENT',
+ else: null
+ }
+ }
}
}
},
@@ -89,7 +130,7 @@ ActionSchema.statics.getActionSummaries = function(item_ids) {
item_type: '$item_type',
// set the current user to false here
- current_user: {$literal: false}
+ current_user: '$current_user'
}
}
])
diff --git a/models/asset.js b/models/asset.js
index 1092cd3e0..b5321cec3 100644
--- a/models/asset.js
+++ b/models/asset.js
@@ -3,7 +3,6 @@ const uuid = require('uuid');
const Schema = mongoose.Schema;
const AssetSchema = new Schema({
-
id: {
type: String,
default: uuid.v4,
@@ -19,12 +18,18 @@ const AssetSchema = new Schema({
type: String,
default: 'article'
},
- headline: String,
- summary: String,
+ scraped: {
+ type: Date,
+ default: null
+ },
+ title: String,
+ description: String,
+ image: String,
section: String,
subsection: String,
- authors: [String],
- publication_date: Date
+ author: String,
+ publication_date: Date,
+ modified_date: Date
}, {
versionKey: false,
timestamps: {
@@ -36,80 +41,42 @@ const AssetSchema = new Schema({
/**
* Search for assets. Currently only returns all.
*/
-AssetSchema.statics.search = function(query) {
-
- return Asset.find(query).exec();
-
-};
+AssetSchema.statics.search = (query) => Asset.find(query);
/**
* Finds an asset by its id.
* @param {String} id identifier of the asset (uuid).
*/
-AssetSchema.statics.findById = function(id) {
-
- return Asset.findOne({id}).exec();
-
-};
+AssetSchema.statics.findById = (id) => Asset.findOne({id});
/**
* Finds a asset by its url.
* @param {String} url identifier of the asset (uuid).
*/
-AssetSchema.statics.findByUrl = function(url) {
-
- return Asset.findOne({'url': url}).exec();
-
-};
+AssetSchema.statics.findByUrl = (url) => Asset.findOne({url});
/**
* Finds a asset by its url.
+ *
+ * NOTE: This function has scalability concerns regarding mongoose's decision
+ * always write {updated_at: new Date()} on every call to findOneAndUpdate
+ * even though the update document exactly matches the query document... In
+ * the future this function should never update, only findOneAndCreate but this
+ * is not possible with the mongoose driver.
+ *
* @param {String} url identifier of the asset (uuid).
*/
-AssetSchema.statics.findOrCreateByUrl = function(url) {
+AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, {
- return Asset.findOne({url})
- .then((asset) => asset ? asset
- : Asset.upsert({url}));
-};
+ // Ensure that if it's new, we return the new object created.
+ new: true,
-/**
- * Upserts an asset.
-*/
-AssetSchema.statics.upsert = function(data) {
- // If an id is not sent, create one.
- if (typeof data.id === 'undefined') {
- data.id = uuid.v4();
- }
+ // Perform an upsert in the event that this doesn't exist.
+ upsert: true,
- // Perform the upsert against the id field.
- let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec()
- .then(() => {
-
- // Pull the freshly minted asset out and return.
- return Asset.findById(data.id);
-
- })
- .catch((err) => {
-
- console.error('Error upserting asset.', err);
- //return new Promise(); // ??? what do we return on error?
-
- });
-
- return updatePromise;
-
-};
-
-/**
- * Remove assets from the db.
- * @param {String} query bson query to identify assets to be removed.
-*/
-AssetSchema.statics.removeAll = function(query) {
-
- return Asset.remove(query).exec();
-
-};
+ // Set the default values if not provided based on the mongoose models.
+ setDefaultsOnInsert: true
+});
const Asset = mongoose.model('Asset', AssetSchema);
diff --git a/models/comment.js b/models/comment.js
index 526f14eef..f03d1b12d 100644
--- a/models/comment.js
+++ b/models/comment.js
@@ -173,20 +173,17 @@ CommentSchema.statics.changeStatus = function(id, status) {
/**
* Add an action to the comment.
- * @param {String} id identifier of the comment (uuid)
+ * @param {String} item_id identifier of the comment (uuid)
+ * @param {String} user_id user id of the action (uuid)
* @param {String} action the new action to the comment
* @return {Promise}
*/
-CommentSchema.statics.addAction = function(id, user_id, action_type) {
- // check that the comment exist
- let action = new Action({
- action_type: action_type,
- item_type: 'comment',
- item_id: id,
- user_id: user_id
- });
- return action.save();
-};
+CommentSchema.statics.addAction = (item_id, user_id, action_type) => Action.insertUserAction({
+ item_id,
+ item_type: 'comment',
+ user_id,
+ action_type
+});
//==============================================================================
// Remove Statics
diff --git a/package.json b/package.json
index 9300c347f..bc2f72f20 100644
--- a/package.json
+++ b/package.json
@@ -4,14 +4,14 @@
"description": "A commenting platform from The Coral Project. https://coralproject.net",
"main": "app.js",
"scripts": {
- "start": "./bin/www",
+ "start": "./bin/cli serve --jobs",
"build": "NODE_ENV=production webpack --config webpack.config.js --bail",
"build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch",
"lint": "eslint bin/* .",
"lint-fix": "eslint . --fix",
- "test": "mocha --compilers js:babel-core/register --recursive tests",
- "test-watch": "mocha --compilers js:babel-core/register --recursive -w tests",
- "embed-start": "NODE_ENV=development npm run build && ./bin/www"
+ "test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests",
+ "test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests",
+ "embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs"
},
"config": {
"pre-git": {
@@ -45,14 +45,16 @@
"express-session": "^1.14.2",
"helmet": "^3.1.0",
"jsonwebtoken": "^7.1.9",
+ "kue": "^0.11.5",
"lodash": "^4.16.6",
+ "metascraper": "^1.0.6",
"mongoose": "^4.6.5",
"morgan": "^1.7.0",
+ "natural": "^0.4.0",
"nodemailer": "^2.6.4",
"passport": "^0.3.2",
"passport-facebook": "^2.1.1",
"passport-local": "^1.0.0",
- "natural": "^0.4.0",
"prompt": "^1.0.0",
"react-linkify": "^0.1.3",
"redis": "^2.6.3",
diff --git a/routes/admin/index.js b/routes/admin/index.js
index 9d3cbd0a9..03852c375 100644
--- a/routes/admin/index.js
+++ b/routes/admin/index.js
@@ -1,14 +1,10 @@
const express = require('express');
const router = express.Router();
-router.get('/embed/stream/preview', (req, res) => {
- res.render('embed-stream', {basePath: '/client/embed/stream'});
-});
-
-// this route is expecting there to be a token in the hash
-// see /views/password-reset-email.ejs
+// Get /password-reset expects a signed token (JWT) in the hash.
+// Links to this endpoint are generated by /views/password-reset-email.ejs.
router.get('/password-reset', (req, res, next) => {
- // TODO: store the redirect uri in the token or something fancy
+ // TODO: store the redirect uri in the token or something fancy.
// admins and regular users should probably be redirected to different places.
res.render('password-reset', {redirectUri: process.env.TALK_ROOT_URL});
});
diff --git a/routes/api/actions/index.js b/routes/api/actions/index.js
index f2f9be390..9724f5a73 100644
--- a/routes/api/actions/index.js
+++ b/routes/api/actions/index.js
@@ -6,7 +6,8 @@ const router = express.Router();
router.delete('/:action_id', (req, res, next) => {
Action
.findOneAndRemove({
- id: req.params.action_id
+ id: req.params.action_id,
+ user_id: req.user.id
})
.then(() => {
res.status(204).end();
diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js
index b261da883..18fd0b7ec 100644
--- a/routes/api/asset/index.js
+++ b/routes/api/asset/index.js
@@ -1,44 +1,81 @@
const express = require('express');
const router = express.Router();
-const Asset = require('../../../models/asset');
-// Search assets.
+const Asset = require('../../../models/asset');
+const scraper = require('../../../services/scraper');
+
+// List assets.
router.get('/', (req, res, next) => {
- let query = {};
+ const {
+ limit = 20,
+ skip = 0,
+ sort = 'asc',
+ field = 'created_at'
+ } = req.query;
- if (typeof req.query.url !== 'undefined') {
- query.url = req.query.url;
- }
+ // Find all the assets.
+ Promise.all([
+ Asset
+ .find({})
+ .sort({[field]: (sort === 'asc') ? 1 : -1})
+ .skip(skip)
+ .limit(limit),
+ Asset.count()
+ ])
+ .then(([result, count]) => {
- Asset.search(query)
- .then((asset) => {
- res.json(asset);
- })
- .catch(next);
+ // Send back the asset data.
+ res.json({
+ result,
+ count
+ });
+ })
+ .catch((err) => {
+ next(err);
+ });
});
-// Get an asset by id
-router.get('/:id', (req, res, next) => {
+// Get an asset by id.
+router.get('/:asset_id', (req, res, next) => {
- Asset.findById(req.params.id)
+ // Send back the asset.
+ Asset
+ .findById(req.params.asset_id)
.then((asset) => {
+ if (!asset) {
+ return res.status(404).end();
+ }
+
res.json(asset);
})
- .catch(next);
-
+ .catch((err) => {
+ next(err);
+ });
});
-// Upsert an asset and return the affected document.
-router.put('/', (req, res, next) => {
+// Adds the asset id to the queue to be scraped.
+router.post('/:asset_id/scrape', (req, res, next) => {
- Asset.upsert(req.body)
+ // Create a new asset scrape job.
+ Asset
+ .findById(req.params.asset_id)
.then((asset) => {
- res.json(asset);
- })
- .catch(next);
+ if (!asset) {
+ return res.status(404).end();
+ }
+ return scraper.create(asset);
+ })
+ .then((job) => {
+
+ // Send the job back for monitoring.
+ res.status(201).json(job);
+ })
+ .catch((err) => {
+ next(err);
+ });
});
module.exports = router;
diff --git a/routes/api/auth/index.js b/routes/api/auth/index.js
index ae6fba01c..369e5ec77 100644
--- a/routes/api/auth/index.js
+++ b/routes/api/auth/index.js
@@ -7,14 +7,25 @@ const router = express.Router();
/**
* This returns the user if they are logged in.
*/
-router.get('/', authorization.needed(), (req, res) => {
+router.get('/', (req, res, next) => {
+ if (req.user) {
+ return next();
+ }
+
+ // When there is no user on the request, then just send back a 204 to this
+ // request. It's not really "an error" if what they asked for isn't available,
+ // but it could be.
+ res.status(204).end();
+}, (req, res) => {
+
+ // Send back the user object.
res.json(req.user.toObject());
});
/**
* This destroys the session of a user, if they have one.
*/
-router.delete('/', (req, res) => {
+router.delete('/', authorization.needed(), (req, res) => {
req.session.destroy(() => {
res.status(204).end();
});
diff --git a/routes/api/comments/index.js b/routes/api/comments/index.js
index 6b38e5f2f..0545a8163 100644
--- a/routes/api/comments/index.js
+++ b/routes/api/comments/index.js
@@ -1,10 +1,11 @@
const express = require('express');
const Comment = require('../../../models/comment');
const wordlist = require('../../../services/wordlist');
+const authorization = require('../../../middleware/authorization');
const router = express.Router();
-router.get('/', (req, res, next) => {
+router.get('/', authorization.needed('admin'), (req, res, next) => {
let query;
if (req.query.status) {
@@ -28,8 +29,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
const {
body,
asset_id,
- parent_id,
- author_id
+ parent_id
} = req.body;
Comment
@@ -38,7 +38,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
asset_id,
parent_id,
status: req.wordlist.matched ? 'rejected' : '',
- author_id
+ author_id: req.user.id
})
.then((comment) => {
@@ -49,7 +49,7 @@ router.post('/', wordlist.filter('body'), (req, res, next) => {
});
});
-router.get('/:comment_id', (req, res, next) => {
+router.get('/:comment_id', authorization.needed('admin'), (req, res, next) => {
Comment
.findById(req.params.comment_id)
.then(comment => {
@@ -65,7 +65,7 @@ router.get('/:comment_id', (req, res, next) => {
});
});
-router.delete('/:comment_id', (req, res, next) => {
+router.delete('/:comment_id', authorization.needed('admin'), (req, res, next) => {
Comment
.removeById(req.params.comment_id)
.then(() => {
@@ -76,7 +76,8 @@ router.delete('/:comment_id', (req, res, next) => {
});
});
-router.put('/:comment_id/status', (req, res, next) => {
+
+router.put('/:comment_id/status', authorization.needed('admin'), (req, res, next) => {
const {
status
} = req.body;
@@ -94,12 +95,11 @@ router.put('/:comment_id/status', (req, res, next) => {
router.post('/:comment_id/actions', (req, res, next) => {
const {
- user_id,
action_type
} = req.body;
Comment
- .addAction(req.params.comment_id, user_id, action_type)
+ .addAction(req.params.comment_id, req.user.id, action_type)
.then((action) => {
res.status(201).json(action);
})
diff --git a/routes/api/index.js b/routes/api/index.js
index 7192a1324..8da3f791b 100644
--- a/routes/api/index.js
+++ b/routes/api/index.js
@@ -1,14 +1,20 @@
const express = require('express');
+const authorization = require('../../middleware/authorization');
const router = express.Router();
-router.use('/asset', require('./asset'));
+router.use('/asset', authorization.needed('admin'), require('./asset'));
+router.use('/settings', authorization.needed('admin'), require('./settings'));
+router.use('/queue', authorization.needed('admin'), require('./queue'));
+
+router.use('/comments', authorization.needed(), require('./comments'));
+router.use('/actions', authorization.needed(), require('./actions'));
+
router.use('/auth', require('./auth'));
-router.use('/comments', require('./comments'));
-router.use('/queue', require('./queue'));
-router.use('/settings', require('./settings'));
router.use('/stream', require('./stream'));
router.use('/user', require('./user'));
-router.use('/actions', require('./actions'));
+
+// Bind the kue handler to the /kue path.
+router.use('/kue', authorization.needed('admin'), require('kue').app);
module.exports = router;
diff --git a/routes/api/queue/index.js b/routes/api/queue/index.js
index f8ab95d6b..f661992f1 100644
--- a/routes/api/queue/index.js
+++ b/routes/api/queue/index.js
@@ -1,6 +1,5 @@
const express = require('express');
const Comment = require('../../../models/comment');
-
const Setting = require('../../../models/setting');
const router = express.Router();
diff --git a/routes/api/settings/index.js b/routes/api/settings/index.js
index 2665cacc8..6e8c64d4f 100644
--- a/routes/api/settings/index.js
+++ b/routes/api/settings/index.js
@@ -1,7 +1,8 @@
-const _ = require('lodash');
const express = require('express');
-const router = express.Router();
const Setting = require('../../../models/setting');
+const _ = require('lodash');
+
+const router = express.Router();
router.get('/', (req, res, next) => {
Setting
diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js
index acbfe3d77..0ef97aad2 100644
--- a/routes/api/stream/index.js
+++ b/routes/api/stream/index.js
@@ -1,52 +1,99 @@
const express = require('express');
const _ = require('lodash');
+const scraper = require('../../../services/scraper');
const Comment = require('../../../models/comment');
const User = require('../../../models/user');
const Action = require('../../../models/action');
const Asset = require('../../../models/asset');
-
const Setting = require('../../../models/setting');
const router = express.Router();
-// Find all the comments by a specific asset_url.
-// . if pre: get the comments that are accepted.
-// . if post: get the comments that are new and accepted.
router.get('/', (req, res, next) => {
// Get the asset_id for this url (or create it if it doesn't exist)
Promise.all([
- Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)),
+
+ // Find or create the asset by url.
+ Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url))
+
+ // Add the found asset to the scraper if it's not already scraped.
+ .then((asset) => {
+ if (!asset.scraped) {
+ return scraper.create(asset).then(() => asset);
+ }
+
+ return asset;
+ }),
+
+ // Get the moderation setting from the settings.
Setting.getModerationSetting()
])
.then(([asset, {moderation}]) => {
- // Get the sitewide moderation setting and return the appropriate comments
- switch(moderation){
- case 'pre':
- return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]);
- case 'post':
- return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]);
- default:
- return Promise.reject(new Error('Moderation setting not found.'));
+ let comments;
+
+ if (moderation === 'post') {
+ comments = Comment.findAcceptedByAssetId(asset.id);
+ } else {
+
+ // Defaults to 'pre' moderation.
+ comments = Comment.findAcceptedAndNewByAssetId(asset.id);
}
+
+ return Promise.all([
+
+ // This is the promised component... Fetch the comments based on the
+ // moderation settings.
+ comments,
+
+ // Send back the reference to the asset.
+ asset
+ ]);
})
// Get all the users and actions for those comments.
.then(([comments, asset]) => {
+
+ // Get the user id's from the author id's as a unique array that gets
+ // sorted.
+ let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort();
+
+ // Fetch the users for which there is a comment available for them.
+ let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : [];
+
+ // Fetch the actions for pretty much everything at this point.
+ let actions = Action.getActionSummaries(_.uniq([
+
+ // Actions can be on assets...
+ asset.id,
+
+ // Comments...
+ ...comments.map((comment) => comment.id),
+
+ // Or Authors...
+ ...userIDs
+ ]), req.user ? req.user.id : false);
+
return Promise.all([
- [asset],
+
+ // Pass back the asset that we loaded...
+ asset,
+
+ // It's comments...
comments,
- User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))),
- Action.getActionSummaries(_.uniq([
- asset.id,
- ...comments.map((comment) => comment.id),
- ...comments.map((comment) => comment.author_id)
- ]))
+
+ // All the users/authors of those comments...
+ users,
+
+ // And all actions about the asset, comments, and users.
+ actions
]);
})
- .then(([assets, comments, users, actions]) => {
+ .then(([asset, comments, users, actions]) => {
+
+ // Send back the payload containing all this data.
res.json({
- assets,
+ assets: [asset],
comments,
users,
actions
diff --git a/routes/api/user/index.js b/routes/api/user/index.js
index 32caa8f52..bbb8c071b 100644
--- a/routes/api/user/index.js
+++ b/routes/api/user/index.js
@@ -7,15 +7,16 @@ const fs = require('fs');
const path = require('path');
const resetEmailFile = fs.readFileSync(path.resolve(__dirname, '../../../views/password-reset-email.ejs'));
const resetEmailTemplate = ejs.compile(resetEmailFile.toString());
+const authorization = require('../../../middleware/authorization');
-router.get('/', (req, res, next) => {
+router.get('/', authorization.needed('admin'), (req, res, next) => {
const {
value = '',
field = 'created_at',
page = 1,
asc = 'false',
limit = 50 // Total Per Page
- } = req.query;
+ } = req.query;
Promise.all([
User
@@ -50,7 +51,7 @@ router.get('/', (req, res, next) => {
.catch(next);
});
-router.post('/:user_id/role', (req, res, next) => {
+router.post('/:user_id/role', authorization.needed('admin'), (req, res, next) => {
User
.addRoleToUser(req.params.user_id, req.body.role)
.then(role => {
@@ -137,9 +138,10 @@ router.post('/request-password-reset', (req, res, next) => {
return mailer.sendSimple(options);
})
.then(() => {
+
// we want to send a 204 regardless of the user being found in the db
// if we fail on missing emails, it would reveal if people are registered or not.
- res.status(204).send('OK');
+ res.status(204).end();
})
.catch(error => {
const errorMsg = typeof error === 'string' ? error : error.message;
diff --git a/routes/index.js b/routes/index.js
index 0c03edcc5..a875d5127 100644
--- a/routes/index.js
+++ b/routes/index.js
@@ -6,11 +6,11 @@ router.use('/admin', require('./admin'));
router.use('/embed', require('./embed'));
router.get('/', (req, res) => {
- return res.render('article', {title: 'Coral Talk'});
+ res.render('article', {title: 'Coral Talk'});
});
router.get('/assets/:asset_title', (req, res) => {
- return res.render('article', {title: req.params.asset_title.split('-').join(' ')});
+ res.render('article', {title: req.params.asset_title.split('-').join(' ')});
});
module.exports = router;
diff --git a/services/scraper.js b/services/scraper.js
new file mode 100644
index 000000000..665d4fc66
--- /dev/null
+++ b/services/scraper.js
@@ -0,0 +1,140 @@
+const kue = require('kue');
+const queue = kue.createQueue();
+const debug = require('debug')('talk:services:scraper');
+const Asset = require('../models/asset');
+const JOB_NAME = 'scraper';
+
+const metascraper = require('metascraper');
+
+/**
+ * Exposes a service object to allow operations to execute against the scraper.
+ * @type {Object}
+ */
+const scraper = {
+
+ /**
+ * creates a new scraper job and scrapes the url when it gets processed.
+ */
+ create(asset) {
+ return new Promise((resolve, reject) => {
+ debug(`Creating job for Asset[${asset.id}]`);
+
+ let job = queue
+ .create(JOB_NAME, {
+ title: `Scrape for asset ${asset.id}`,
+ asset_id: asset.id
+ })
+ .attempts(10)
+ .delay(1000)
+ .backoff({type: 'exponential'})
+ .save((err) => {
+ if (err) {
+ return reject(err);
+ }
+
+ debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
+
+ return resolve(job);
+ });
+ });
+ },
+
+ /**
+ * Scrapes the given asset for metadata.
+ */
+ scrape(asset) {
+ return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, {
+ section: ($) => $('meta[property="article:section"]').attr('content'),
+ modified: ($) => $('meta[property="article:modified"]').attr('content')
+ }));
+ },
+
+ update(id, meta) {
+ return Asset.update({id}, {
+ $set: {
+ title: meta.title || '',
+ description: meta.description || '',
+ image: meta.image ? meta.image : '',
+ author: meta.author || '',
+ publication_date: meta.date || '',
+ modified_date: meta.modified || '',
+ section: meta.section || '',
+ scraped: new Date()
+ }
+ });
+ },
+
+ /**
+ * Start the queue processor for the scraper job.
+ */
+ process() {
+
+ debug(`Now processing ${JOB_NAME} jobs`);
+
+ // Process jobs with the processJob function.
+ queue.process(JOB_NAME, (job, done) => {
+
+ debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`);
+
+ Asset
+
+ // Find the asset, or complain that it doesn't exist.
+ .findById(job.data.asset_id)
+ .then((asset) => {
+ if (!asset) {
+ throw new Error('asset not found');
+ }
+
+ return asset;
+ })
+
+ // Scrape the metadata from the asset.
+ .then(scraper.scrape)
+
+ // Assign the metadata retrieved for the asset to the db.
+ .then((meta) => {
+ debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`);
+
+ return scraper.update(job.data.asset_id, meta);
+ })
+
+ // Finish the job because we just handled our scraping + updating the
+ // asset in the database.
+ .then(() => {
+ debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`);
+ done();
+ })
+
+ // Handle errors that occur.
+ .catch((err) => {
+ console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err);
+
+ done(err);
+ });
+ });
+ },
+
+ /**
+ * Shuts down the current queue to ensure that the application can shutdown
+ * cleanly.
+ */
+ shutdown() {
+ return new Promise((resolve, reject) => {
+
+ // Shutdown and give the queue 5 seconds to shutdown before we start
+ // killing jobs.
+ queue.shutdown(5000, (err) => {
+ if (err) {
+ return reject(err);
+ }
+
+ debug(`Processing for ${JOB_NAME} jobs stopped`);
+
+ resolve();
+ });
+ });
+ }
+
+};
+
+module.exports = scraper;
diff --git a/swagger.yaml b/swagger.yaml
index fd3b0012b..09adb97d6 100644
--- a/swagger.yaml
+++ b/swagger.yaml
@@ -256,6 +256,86 @@ paths:
description: An error occured.
schema:
$ref: '#/definitions/Error'
+ /asset:
+ get:
+ parameters:
+ - name: limit
+ in: query
+ type: number
+ format: int32
+ description: Limit the listing results
+ - name: skip
+ in: query
+ type: number
+ format: int32
+ description: Skip the listing results
+ - name: sort
+ in: query
+ enum:
+ - asc
+ - desc
+ type: string
+ description: Sorting method
+ - name: field
+ in: query
+ type: string
+ description: Field to sort by.
+ responses:
+ 200:
+ description: Assets listed.
+ schema:
+ type: object
+ properties:
+ count:
+ type: number
+ description: Total number of assets found.
+ result:
+ type: array
+ items:
+ $ref: '#/definitions/Asset'
+
+ /asset/{asset_id}:
+ get:
+ parameters:
+ - name: asset_id
+ in: path
+ required: true
+ type: string
+ format: uuid
+ responses:
+ 200:
+ description: The requested asset.
+ schema:
+ $ref: '#/definitions/Asset'
+ 404:
+ description: The asset was not found.
+ 500:
+ description: An error occured.
+ schema:
+ $ref: '#/definitions/Error'
+
+
+ /asset/{asset_id}/scrape:
+ post:
+ parameters:
+ - name: asset_id
+ in: path
+ required: true
+ type: string
+ format: uuid
+ responses:
+ 201:
+ description: The job that was created.
+ schema:
+ $ref: '#/definitions/Job'
+ 404:
+ description: The asset was not found.
+ 500:
+ description: An error occured.
+ schema:
+ $ref: '#/definitions/Error'
+
+
/stream:
get:
tags:
@@ -420,3 +500,10 @@ definitions:
type: object
Settings:
type: object
+ Job:
+ type: object
+ properties:
+ id:
+ format: number
+ state:
+ format: string
diff --git a/tests/index.js b/tests/index.js
deleted file mode 100644
index 0c3d947ce..000000000
--- a/tests/index.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const expect = require('chai').expect;
-
-describe('Comment', () => {
- describe('#add', () => {
- it('should add a comment', () => {
- expect(0).to.be.equal(0);
- });
- });
-});
diff --git a/tests/models/action.js b/tests/models/action.js
index 78f5fd1d0..c354b8f83 100644
--- a/tests/models/action.js
+++ b/tests/models/action.js
@@ -1,27 +1,27 @@
-require('../utils/mongoose');
-
const Action = require('../../models/action');
const expect = require('chai').expect;
describe('Action: models', () => {
let mockActions;
+
beforeEach(() => {
return Action.create([{
action_type: 'flag',
item_id: '123',
- item_type: 'comments'
+ item_type: 'comment',
+ user_id: 'flagginguserid'
}, {
action_type: 'flag',
item_id: '456',
- item_type: 'comments'
+ item_type: 'comment'
}, {
action_type: 'flag',
item_id: '123',
- item_type: 'comments'
+ item_type: 'comment'
}, {
action_type: 'like',
item_id: '123',
- item_type: 'comments'
+ item_type: 'comment'
}]).then((actions) => {
mockActions = actions;
});
@@ -30,8 +30,7 @@ describe('Action: models', () => {
describe('#findById()', () => {
it('should find an action by id', () => {
return Action.findById(mockActions[0].id).then((result) => {
- expect(result).to.have.property('action_type')
- .and.to.equal('flag');
+ expect(result).to.have.property('action_type', 'flag');
});
});
});
@@ -46,27 +45,55 @@ describe('Action: models', () => {
describe('#getActionSummaries()', () => {
it('should return properly formatted summaries from an array of item_ids', () => {
- return Action.getActionSummaries(['123', '789']).then((result) => {
- expect(result).to.have.length(2);
+ return Action.getActionSummaries(['123', '789']).then((summaries) => {
+ expect(summaries).to.have.length(2);
- const sorted = result.sort((a, b) => a.count - b.count);
-
- expect(sorted[0]).to.deep.equal({
+ expect(summaries).to.deep.include({
action_type: 'like',
count: 1,
item_id: '123',
- item_type: 'comments',
- current_user: false
+ item_type: 'comment',
+ current_user: null
});
- expect(sorted[1]).to.deep.equal({
+ expect(summaries).to.deep.include({
action_type: 'flag',
count: 2,
item_id: '123',
- item_type: 'comments',
- current_user: false
+ item_type: 'comment',
+ current_user: null
});
});
});
+
+ it('should include a current user when one is passed', () => {
+ return Action
+ .getActionSummaries(['123'], 'flagginguserid')
+ .then((summaries) => {
+ expect(summaries).to.have.length(2);
+
+ let summary = summaries.find((s) => s.item_id === '123' && s.action_type === 'flag');
+
+ expect(summary).to.not.be.undefined;
+ expect(summary.current_user).to.not.be.null;
+ expect(summary.current_user).to.have.property('item_id', '123');
+ expect(summary.current_user).to.have.property('item_type', 'comment');
+ expect(summary.current_user).to.have.property('user_id', 'flagginguserid');
+ expect(summary.current_user).to.have.property('action_type', 'flag');
+ });
+ });
+
+ it('should not include a current user when one is passed for a user that doesn\'t have an action', () => {
+ return Action
+ .getActionSummaries(['123'], 'flagginguserid2')
+ .then((summaries) => {
+ expect(summaries).to.have.length(2);
+
+ summaries.forEach((summary) => {
+ expect(summary).to.not.be.undefined;
+ expect(summary).to.have.property('current_user', null);
+ });
+ });
+ });
});
});
diff --git a/tests/models/asset.js b/tests/models/asset.js
index d0c3d3429..a5c4ae62b 100644
--- a/tests/models/asset.js
+++ b/tests/models/asset.js
@@ -1,7 +1,3 @@
-/* eslint-env node, mocha */
-
-require('../utils/mongoose');
-
const Asset = require('../../models/asset');
const expect = require('chai').expect;
@@ -74,35 +70,4 @@ describe('Asset: model', () => {
});
});
});
-
- describe('#upsert', ()=> {
- it('should insert an asset with no id', () => {
- return Asset.upsert({url: 'http://newasset.test.com'})
- .then((asset) => {
- expect(asset).to.have.property('id');
- });
- });
-
- it('should update an asset when the id exists', () => {
- return Asset.upsert({id: 1, url: 'http://new.test.com'})
- .then((asset) => {
- expect(asset).to.have.property('id')
- .and.to.equal('1');
- expect(asset).to.have.property('url')
- .and.to.equal('http://new.test.com');
- });
- });
- });
-
- describe('#removeAll', ()=> {
- it('should insert an asset with no id', () => {
- return Asset.removeAll({id:1})
- .then(() => {
- return Asset.findById(1);
- })
- .then((result) => {
- expect(result).to.be.null;
- });
- });
- });
});
diff --git a/tests/models/comment.js b/tests/models/comment.js
index f8dd7f9f5..07834a186 100644
--- a/tests/models/comment.js
+++ b/tests/models/comment.js
@@ -1,5 +1,3 @@
-require('../utils/mongoose');
-
const Comment = require('../../models/comment');
const User = require('../../models/user');
const Action = require('../../models/action');
diff --git a/tests/models/setting.js b/tests/models/setting.js
index 3f4dc27fa..a36b363fb 100644
--- a/tests/models/setting.js
+++ b/tests/models/setting.js
@@ -1,7 +1,3 @@
-/* eslint-env node, mocha */
-
-require('../utils/mongoose');
-
const Setting = require('../../models/setting');
const expect = require('chai').expect;
diff --git a/tests/models/user.js b/tests/models/user.js
index d0886a2e8..618472881 100644
--- a/tests/models/user.js
+++ b/tests/models/user.js
@@ -1,5 +1,3 @@
-require('../utils/mongoose');
-
const User = require('../../models/user');
const Comment = require('../../models/comment');
const expect = require('chai').expect;
diff --git a/tests/utils/mongoose.js b/tests/mongoose.js
similarity index 71%
rename from tests/utils/mongoose.js
rename to tests/mongoose.js
index 1549440a6..9d3ba1195 100644
--- a/tests/utils/mongoose.js
+++ b/tests/mongoose.js
@@ -1,8 +1,4 @@
-const mongoose = require('../../mongoose');
-
-// Ensure the NODE_ENV is set to 'test',
-// this is helpful when you would like to change behavior when testing.
-process.env.NODE_ENV = 'test';
+const mongoose = require('../mongoose');
beforeEach(function (done) {
function clearDB() {
diff --git a/tests/passport.js b/tests/passport.js
new file mode 100644
index 000000000..2d1c53ab9
--- /dev/null
+++ b/tests/passport.js
@@ -0,0 +1,25 @@
+const authorization = require('../middleware/authorization');
+
+// Add the passport middleware here before it's setup.
+authorization.middleware.push((req, res, next) => {
+ req.user = JSON.parse(new Buffer(req.get('X-Mock-Authorization'), 'base64').toString('ascii'));
+
+ next();
+});
+
+const MockStrategy = {
+
+ /**
+ * Injects the new user into the request header for the mock middleware to
+ * interpret.
+ * @param {Object} user the user to inject
+ * @return {Object} the headers to add to the request
+ */
+ inject(user) {
+ return {
+ 'X-Mock-Authorization': new Buffer(JSON.stringify(user)).toString('base64')
+ };
+ }
+};
+
+module.exports = MockStrategy;
diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js
index aa764e214..fb67b7b48 100644
--- a/tests/routes/api/assets/index.js
+++ b/tests/routes/api/assets/index.js
@@ -1,95 +1,10 @@
-require('../../../utils/mongoose');
+describe('/assets', () => {
-const chai = require('chai');
-const expect = chai.expect;
-const server = require('../../../../app');
+ describe('GET', () => {
-// Setup chai.
-chai.should();
-chai.use(require('chai-http'));
+ it('should return assets that we search for');
+ it('should not return assets that we do not search for');
-let fixture = {
- 'url': 'http://hhgg.com/total-perspective-vortex',
- 'type': 'article',
- 'headline': 'The Total Perspective Vortex',
- 'summary': 'You are an insignificant dot on an insignificant dot.',
- 'section': 'Everything',
- 'authors': ['Ford Prefect']
-};
-
-describe('Asset: routes', () => {
-
- describe('/GET Asset', () => {
- describe('#get', () => {
- it('It should get an empty array when there are no assets.', (done) => {
-
- chai.request(server)
- .get('/api/v1/asset')
- .end((err, res) => {
-
- if (err) {
- throw new Error(err);
- }
-
- res.should.have.status(200);
- res.body.should.be.a('array');
- res.body.length.should.be.eql(0);
- done();
- });
-
- });
- });
});
- // This test checks PUT and read
- describe('/PUT Asset', () => {
- describe('#put', () => {
- it('It should save an asset and load it again.', (done) => {
-
- chai.request(server)
- .put('/api/v1/asset')
- .send(fixture)
- .end((err, res) => {
-
- if (err) {
- throw new Error(err);
- }
-
- res.should.have.status(200);
- res.body.should.be.a('object');
-
- // Id should be generated by the model if absent.
- res.body.should.have.property('id');
-
- // Save the asset id to compare with GET result.
- let assetId = res.body.id;
-
- // Load the asset to make sure it's really there.
- chai.request(server)
- .get(`/api/v1/asset?url=${encodeURIComponent(fixture.url)}`)
- .end((err, res) => {
-
- if (err) {
- throw new Error(err);
- }
-
- res.should.have.status(200);
- res.body.should.be.an('array');
-
- let asset = res.body[0];
-
- expect(asset).to.have.property('id');
-
- // Ensure the asset has the same id as above.
- // This tests the single url per Id concept.
- expect(assetId).to.equal(asset.id);
-
- done();
-
- });
- });
- });
- });
- }); // End describe /PUT Asset
-
});
diff --git a/tests/routes/api/auth/index.js b/tests/routes/api/auth/index.js
index dd49a1472..ddca63fe8 100644
--- a/tests/routes/api/auth/index.js
+++ b/tests/routes/api/auth/index.js
@@ -1,5 +1,3 @@
-require('../../../utils/mongoose');
-
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js
index ec9e05d03..ff7f3a9d4 100644
--- a/tests/routes/api/comments/index.js
+++ b/tests/routes/api/comments/index.js
@@ -1,6 +1,4 @@
-process.env.NODE_ENV = 'test';
-
-require('../../../utils/mongoose');
+const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
@@ -68,6 +66,7 @@ describe('Get /comments', () => {
it('should return all the comments', () => {
return chai.request(app)
.get('/api/v1/comments')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
@@ -126,6 +125,7 @@ describe('Get comments by status and action', () => {
it('should return all the rejected comments', () => {
return chai.request(app)
.get('/api/v1/comments?status=rejected')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
expect(res.body[0]).to.have.property('id', 'abc');
@@ -135,6 +135,7 @@ describe('Get comments by status and action', () => {
it('should return all the approved comments', () => {
return chai.request(app)
.get('/api/v1/comments?status=accepted')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
expect(res.body[0]).to.have.property('id', 'hij');
@@ -144,6 +145,7 @@ describe('Get comments by status and action', () => {
it('should return all the new comments', () => {
return chai.request(app)
.get('/api/v1/comments?status=new')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
expect(res.body[0]).to.have.property('id', 'def');
@@ -153,6 +155,7 @@ describe('Get comments by status and action', () => {
it('should return all the flagged comments', () => {
return chai.request(app)
.get('/api/v1/comments?action_type=flag')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
@@ -195,6 +198,7 @@ describe('Post /comments', () => {
it('should create a comment', () => {
return chai.request(app)
.post('/api/v1/comments')
+ .set(passport.inject({roles: []}))
.send({'body': 'Something body.', 'author_id': '123', 'asset_id': '1', 'parent_id': ''})
.then((res) => {
expect(res).to.have.status(201);
@@ -205,6 +209,7 @@ describe('Post /comments', () => {
it('should create a comment with a rejected status if it contains a bad word', () => {
return chai.request(app)
.post('/api/v1/comments')
+ .set(passport.inject({roles: []}))
.send({'body': 'bad words are the baddest', 'author_id': '123', 'asset_id': '1', 'parent_id': ''})
.then((res) => {
expect(res).to.have.status(201);
@@ -262,6 +267,7 @@ describe('Get /:comment_id', () => {
it('should return the right comment for the comment_id', () => {
return chai.request(app)
.get('/api/v1/comments/abc')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(200);
expect(res).to.have.property('body');
@@ -318,6 +324,7 @@ describe('Remove /:comment_id', () => {
it('it should remove comment', () => {
return chai.request(app)
.delete('/api/v1/comments/abc')
+ .set(passport.inject({roles: ['admin']}))
.then((res) => {
expect(res).to.have.status(204);
@@ -329,11 +336,6 @@ describe('Remove /:comment_id', () => {
});
});
-process.on('unhandledRejection', (reason) => {
- console.error('Reason: ');
- console.error(reason);
-});
-
describe('Put /:comment_id/status', () => {
const comments = [{
@@ -384,12 +386,26 @@ describe('Put /:comment_id/status', () => {
it('it should update status', function() {
return chai.request(app)
.put('/api/v1/comments/abc/status')
+ .set(passport.inject({roles: ['admin']}))
.send({status: 'accepted'})
.then((res) => {
expect(res).to.have.status(204);
expect(res.body).to.be.empty;
});
});
+
+ it('it should not allow a non-admin to update status', () => {
+ return chai.request(app)
+ .put('/api/v1/comments/abc/status')
+ .set(passport.inject({roles: []}))
+ .send({status: 'accepted'})
+ .then((res) => {
+ expect(res).to.be.empty;
+ })
+ .catch((err) => {
+ expect(err).to.have.property('status', 401);
+ });
+ });
});
describe('Post /:comment_id/actions', () => {
@@ -442,6 +458,7 @@ describe('Post /:comment_id/actions', () => {
it('it should update actions', () => {
return chai.request(app)
.post('/api/v1/comments/abc/actions')
+ .set(passport.inject({id: '456', roles: ['admin']}))
.send({'user_id': '456', 'action_type': 'flag'})
.then((res) => {
expect(res).to.have.status(201);
diff --git a/tests/routes/api/queue/index.js b/tests/routes/api/queue/index.js
index f21fe1331..74137a49a 100644
--- a/tests/routes/api/queue/index.js
+++ b/tests/routes/api/queue/index.js
@@ -1,6 +1,4 @@
-process.env.NODE_ENV = 'test';
-
-require('../../../utils/mongoose');
+const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
@@ -71,6 +69,7 @@ describe('Get moderation queues rejected, pending, flags', () => {
it('should return all the pending comments', function(done){
chai.request(app)
.get('/api/v1/queue/comments/pending')
+ .set(passport.inject({roles: ['admin']}))
.end(function(err, res){
expect(err).to.be.null;
expect(res).to.have.status(200);
diff --git a/tests/routes/api/settings/index.js b/tests/routes/api/settings/index.js
index 41c37828e..9f4466a7f 100644
--- a/tests/routes/api/settings/index.js
+++ b/tests/routes/api/settings/index.js
@@ -1,13 +1,12 @@
-process.env.NODE_ENV = 'test';
-
-require('../../../utils/mongoose');
+const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
-const chaiHttp = require('chai-http');
-chai.use(chaiHttp);
const expect = chai.expect;
+chai.should();
+chai.use(require('chai-http'));
+
const Setting = require('../../../../models/setting');
const defaults = {id: '1', moderation: 'pre'};
@@ -17,15 +16,16 @@ describe('GET /settings', () => {
return Setting.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true});
});
- it('should return a settings object', done => {
- chai.request(app)
+ it('should return a settings object', () => {
+ return chai.request(app)
.get('/api/v1/settings')
- .end((err, res) => {
- expect(err).to.be.null;
+ .set(passport.inject({
+ roles: ['admin']
+ }))
+ .then((res) => {
expect(res).to.have.status(200);
expect(res).to.be.json;
expect(res.body).to.have.property('moderation', 'pre');
- done(err);
});
});
});
@@ -33,25 +33,26 @@ describe('GET /settings', () => {
// update the settings.
describe('update settings', () => {
it('should respond ok to a PUT', () => {
- return Setting.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true})
- .then(() => {
- return chai.request(app)
- .put('/api/v1/settings')
- .send({moderation: 'post'})
- .then(res => {
- expect(res).to.have.status(204);
+ return Setting
+ .update({id: '1'}, {$setOnInsert: defaults}, {upsert: true})
+ .then(() => {
+ return chai.request(app)
+ .put('/api/v1/settings')
+ .set(passport.inject({
+ roles: ['admin']
+ }))
+ .send({moderation: 'post'});
+ })
+ .then(res => {
+ expect(res).to.have.status(204);
- return Setting.getSettings();
+ return Setting.getSettings();
+ })
+ .then(settings => {
- })
- .then(settings => {
- // confirm updated settings in db
- expect(settings).to.have.property('moderation');
- expect(settings.moderation).to.equal('post');
- })
- .catch(err => {
- throw err;
- });
- });
+ // confirm updated settings in db
+ expect(settings).to.have.property('moderation');
+ expect(settings.moderation).to.equal('post');
+ });
});
});
diff --git a/tests/routes/api/stream/index.js b/tests/routes/api/stream/index.js
index 009184b6b..7d05c9708 100644
--- a/tests/routes/api/stream/index.js
+++ b/tests/routes/api/stream/index.js
@@ -1,5 +1,3 @@
-require('../../../utils/mongoose');
-
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
@@ -92,8 +90,8 @@ describe('api/stream: routes', () => {
.then(res => {
expect(res).to.have.status(200);
expect(res.body.assets.length).to.equal(1);
- expect(res.body.comments.length).to.equal(1);
- expect(res.body.users.length).to.equal(1);
+ expect(res.body.comments.length).to.equal(2);
+ expect(res.body.users.length).to.equal(2);
expect(res.body.actions.length).to.equal(1);
});
});
diff --git a/tests/services/scraper.js b/tests/services/scraper.js
new file mode 100644
index 000000000..f3b5d006c
--- /dev/null
+++ b/tests/services/scraper.js
@@ -0,0 +1,22 @@
+describe('scraper: services', () => {
+ describe('#create', () => {
+ it('should create a new kue job');
+ });
+
+ describe('#scrape', () => {
+ it('should scrape complete information');
+ it('should scrape what it can');
+ });
+
+ describe('#update', () => {
+ it('should update the database record entries from the meta');
+ });
+
+ describe('#process', () => {
+ it('should start the processor to scrape assets');
+ });
+
+ describe('#shutdown', () => {
+ it('should shutdown the job processor');
+ });
+});
diff --git a/util.js b/util.js
new file mode 100644
index 000000000..6907d90cd
--- /dev/null
+++ b/util.js
@@ -0,0 +1,42 @@
+const util = module.exports = {};
+
+/**
+ * Stores an array of functions that should be executed in the event that the
+ * application needs to shutdown.
+ * @type {Array}
+ */
+util.toshutdown = [];
+
+/**
+ * Calls all the shutdown functions and then ends the process.
+ * @param {Number} [defaultCode=0] default return code upon sucesfull shutdown.
+ */
+util.shutdown = (defaultCode = 0) => {
+ Promise
+ .all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func))
+ .then(() => {
+ process.exit(defaultCode);
+ })
+ .catch((err) => {
+ console.error(err);
+
+ process.exit(1);
+ });
+};
+
+/**
+ * Waits until an event is triggered by the node runtime and elevates a series
+ * of jobs to be ran in the event we need to shutdown.
+ * @param {Array} jobs Array of promise capable shutdown functions that are
+ * executed.
+ */
+util.onshutdown = (jobs) => {
+
+ // Add the new jobs to shutdown to the object reference.
+ util.toshutdown = util.toshutdown.concat(jobs);
+};
+
+// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the
+// event that we have an external event.
+process.on('SIGTERM', () => util.shutdown());
+process.on('SIGINT', () => util.shutdown());
diff --git a/views/article.ejs b/views/article.ejs
index 7d168f960..71db59593 100644
--- a/views/article.ejs
+++ b/views/article.ejs
@@ -1,8 +1,10 @@
-
+
+
+
diff --git a/views/embed/stream.ejs b/views/embed/stream.ejs
index 5cdef0811..f87318db6 100644
--- a/views/embed/stream.ejs
+++ b/views/embed/stream.ejs
@@ -10,4 +10,4 @@
-
+