diff --git a/.eslintrc.json b/.eslintrc.json index 6ac5a08e6..035a86189 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,25 +5,15 @@ }, "extends": "eslint:recommended", "rules": { - "indent": [ - "error", + "indent": ["error", 2 ], "no-console": [ 0 ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "single" - ], - "semi": [ - "error", - "always" - ], + "linebreak-style": ["error", "unix"], + "quotes": ["error", "single"], + "semi": ["error", "always"], "no-template-curly-in-string": [1], "no-unsafe-negation": [1], "array-callback-return": [1], @@ -35,7 +25,6 @@ "no-throw-literal": [2], "yoda": [1], "no-path-concat": [2], - "no-process-exit": [2], "eol-last": [1], "no-continue": [1], "no-nested-ternary": [1], @@ -46,20 +35,20 @@ "no-const-assign": [2], "no-duplicate-imports": [2], "prefer-template": [1], - "comma-spacing": [ - "error", - { + "comma-spacing": ["error", { "after": true - } - ], + }], "no-var": [2], "no-lonely-if": [2], "curly": [2], - "no-unused-vars": ["error", { "argsIgnorePattern": "next" }], - "no-multiple-empty-lines": [ - "error", - {"max": 1} - ], - "newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }] + "no-unused-vars": ["error", { + "argsIgnorePattern": "next" + }], + "no-multiple-empty-lines": ["error", { + "max": 1 + }], + "newline-per-chained-call": ["error", { + "ignoreChainWithDepth": 2 + }] } } diff --git a/bin/cli b/bin/cli index bec5ea84f..d8148282f 100755 --- a/bin/cli +++ b/bin/cli @@ -19,7 +19,10 @@ const pkg = require('../package.json'); program .version(pkg.version) + .command('serve', 'serve the application') + .command('assets', 'interact with assets') .command('settings', 'work with the application settings') + .command('jobs', 'work with the job queues') .command('users', 'work with the application auth') .parse(process.argv); diff --git a/bin/cli-assets b/bin/cli-assets new file mode 100755 index 000000000..99965c6d9 --- /dev/null +++ b/bin/cli-assets @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const pkg = require('../package.json'); +const Table = require('cli-table'); +const Asset = require('../models/asset'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +// Register the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); + +/** + * Lists all the assets registered in the database. + */ +function listAssets() { + Asset + .find({}) + .sort({'created_at': 1}) + .then((asset) => { + let table = new Table({ + head: [ + 'ID', + 'Title', + 'URL' + ] + }); + + asset.forEach((asset) => { + table.push([ + asset.id, + asset.title ? asset.title : '', + asset.url ? asset.url : '' + ]); + }); + + console.log(table.toString()); + util.shutdown(); + }) + .catch((err) => { + console.error(err); + util.shutdown(1); + }); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .version(pkg.version); + +program + .command('list') + .description('list all the assets in the database') + .action(listAssets); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (!process.argv.slice(2).length) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-jobs b/bin/cli-jobs new file mode 100755 index 000000000..f14276d38 --- /dev/null +++ b/bin/cli-jobs @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const scraper = require('../services/scraper'); +const util = require('../util'); +const mongoose = require('../mongoose'); + +util.onshutdown([ + () => mongoose.disconnect() +]); + +function processJobs() { + + // Start the processor. + scraper.process(); + + // The scraper only needs to shutdown when the scraper has actually been + // started. + util.onshutdown([ + () => scraper.shutdown() + ]); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .command('process') + .description('starts job processing') + .action(processJobs); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (process.argv.length <= 2) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-serve b/bin/cli-serve new file mode 100755 index 000000000..14c1261d7 --- /dev/null +++ b/bin/cli-serve @@ -0,0 +1,132 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +const app = require('../app'); +const debug = require('debug')('talk:server'); +const http = require('http'); +const init = require('../init'); +const scraper = require('../services/scraper'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +/** +* Get port from environment and store in Express. +*/ +const port = normalizePort(process.env.TALK_PORT || '3000'); + +app.set('port', port); + +/** +* Create HTTP server. +*/ +const server = http.createServer(app); + +/** + * Event listener for HTTP server "error" event. + */ +function onError(error) { + if (error.syscall !== 'listen') { + throw error; + } + + let bind = typeof port === 'string' + ? `Pipe ${port}` + : `Port ${port}`; + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + console.error(`${bind} requires elevated privileges`); + break; + case 'EADDRINUSE': + console.error(`${bind} is already in use`); + break; + } + + throw error; +} + +/** + * Normalize a port into a number, string, or false. + */ + +function normalizePort(val) { + let port = parseInt(val, 10); + + if (isNaN(port)) { + // named pipe + return val; + } + + if (port >= 0) { + // port number + return port; + } + + return false; +} + +/** + * Event listener for HTTP server "listening" event. + */ + +function onListening() { + let addr = server.address(); + let bind = typeof addr === 'string' + ? `pipe ${ addr}` + : `port ${ addr.port}`; + debug(`Listening on ${ bind}`); +} + +/** + * Start the app. + */ +function startApp() { + init().then(() => { + + /** + * Listen on provided port, on all network interfaces. + */ + server.listen(port); + server.on('error', onError); + server.on('listening', onListening); + }); +} + +/** + * Module dependencies. + */ + +const program = require('commander'); + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .option('-j, --jobs', 'enable job processing on this thread') + .parse(process.argv); + +// Start the application serving. +startApp(); + +// Enable job processing on the thread if enabled. +if (program.jobs) { + + // Start the processor. + scraper.process(); +} + +// Define a safe shutdown function to call in the event we need to shutdown +// because the node hooks are below which will interrupt the shutdown process. +// Shutdown the mongoose connection, the app server, and the scraper. +util.onshutdown([ + () => program.jobs ? scraper.shutdown() : null, + () => mongoose.disconnect(), + () => server.close() +]); diff --git a/bin/cli-settings b/bin/cli-settings index cff6c04ed..c639a5ca5 100755 --- a/bin/cli-settings +++ b/bin/cli-settings @@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG; */ const program = require('commander'); +const mongoose = require('../mongoose'); +const Setting = require('../models/setting'); +const util = require('../util'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); //============================================================================== // Setting up the program command line arguments. @@ -20,19 +28,17 @@ program .command('init') .description('initilizes the talk settings') .action(() => { - const mongoose = require('../mongoose'); - const Setting = require('../models/setting'); const defaults = {id: '1', moderation: 'pre'}; Setting .update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}) .then(() => { console.log('Created settings object.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(`failed to create the settings object ${JSON.stringify(err)}`); - throw new Error(err); // just to be safe + util.shutdown(1); }); }); @@ -41,4 +47,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/cli-users b/bin/cli-users index 54d7f2927..2e0dc84e6 100755 --- a/bin/cli-users +++ b/bin/cli-users @@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG; const program = require('commander'); const pkg = require('../package.json'); const prompt = require('prompt'); +const User = require('../models/user'); +const mongoose = require('../mongoose'); +const util = require('../util'); +const Table = require('cli-table'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); /** * Prompts for input and registers a user based on those. */ function createUser(options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - return new Promise((resolve, reject) => { if (options.flag_mode) { @@ -74,11 +80,11 @@ function createUser(options) { }) .then((user) => { console.log(`Created user ${user.id}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -86,20 +92,17 @@ function createUser(options) { * Deletes a user. */ function deleteUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .findOneAndRemove({ id: userID }) .then(() => { console.log('Deleted user'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -107,9 +110,6 @@ function deleteUser(userID) { * Changes the password for a user. */ function passwd(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - prompt.start(); prompt.get([ @@ -128,13 +128,13 @@ function passwd(userID) { ], (err, result) => { if (err) { console.error(err); - mongoose.disconnect(); + util.shutdown(); return; } if (result.password !== result.confirmPassword) { console.error(new Error('Password mismatch')); - mongoose.disconnect(); + util.shutdown(1); return; } @@ -142,11 +142,11 @@ function passwd(userID) { .changePassword(userID, result.password) .then(() => { console.log('Password changed.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); }); } @@ -155,9 +155,6 @@ function passwd(userID) { * Updates the user from the options array. */ function updateUser(userID, options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - const updates = []; if (options.email && typeof options.email === 'string' && options.email.length > 0) { @@ -189,11 +186,11 @@ function updateUser(userID, options) { .all(updates.map((q) => q.exec())) .then(() => { console.log(`User ${userID} updated.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -201,10 +198,6 @@ function updateUser(userID, options) { * Lists all the users registered in the database. */ function listUsers() { - const Table = require('cli-table'); - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .all() .then((users) => { @@ -229,11 +222,11 @@ function listUsers() { }); console.log(table.toString()); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -243,18 +236,15 @@ function listUsers() { * @param {String} srcUserID id of the user to which is the source of the merge */ function mergeUsers(dstUserID, srcUserID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .mergeUsers(dstUserID, srcUserID) .then(() => { console.log(`User ${srcUserID} was merged into user ${dstUserID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -264,18 +254,15 @@ function mergeUsers(dstUserID, srcUserID) { * @param {String} role the role to add */ function addRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .addRoleToUser(userID, role) .then(() => { console.log(`Added the ${role} role to User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -285,18 +272,15 @@ function addRole(userID, role) { * @param {String} role the role to remove */ function removeRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .removeRoleFromUser(userID, role) .then(() => { console.log(`Removed the ${role} role from User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -305,18 +289,15 @@ function removeRole(userID, role) { * @param {String} userID the ID of a user to disable */ function disableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .disableUser(userID) .then(() => { console.log(`User ${userID} was disabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -325,18 +306,15 @@ function disableUser(userID) { * @param {String} userID the ID of a user to enable */ function enableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .enableUser(userID) .then(() => { console.log(`User ${userID} was enabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -408,4 +386,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/www b/bin/www deleted file mode 100755 index 3e9e20918..000000000 --- a/bin/www +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env node - -/** - * Setup the debug paramater. - */ - -process.env.DEBUG = process.env.TALK_DEBUG; - -/** - * Module dependencies. - */ - -const app = require('../app'); -const debug = require('debug')('talk:server'); -const http = require('http'); -const init = require('../init'); -const port = normalizePort(process.env.TALK_PORT || '3000'); - -let server; - -init().then(() => { - - /** - * Get port from environment and store in Express. - */ - app.set('port', port); - - /** - * Create HTTP server. - */ - server = http.createServer(app); - - /** - * Listen on provided port, on all network interfaces. - */ - server.listen(port); - server.on('error', onError); - server.on('listening', onListening); -}); - -/** - * Normalize a port into a number, string, or false. - */ - -function normalizePort(val) { - let port = parseInt(val, 10); - - if (isNaN(port)) { - // named pipe - return val; - } - - if (port >= 0) { - // port number - return port; - } - - return false; -} - -/** - * Event listener for HTTP server "error" event. - */ - -function onError(error) { - if (error.syscall !== 'listen') { - throw error; - } - - let bind = typeof port === 'string' - ? `Pipe ${ port}` - : `Port ${ port}`; - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - console.error(`${bind} requires elevated privileges`); - break; - case 'EADDRINUSE': - console.error(`${bind} is already in use`); - break; - } - - throw error; -} - -/** - * Event listener for HTTP server "listening" event. - */ - -function onListening() { - let addr = server.address(); - let bind = typeof addr === 'string' - ? `pipe ${ addr}` - : `port ${ addr.port}`; - debug(`Listening on ${ bind}`); -} diff --git a/client/coral-embed-stream/src/CommentStream.js b/client/coral-embed-stream/src/CommentStream.js index a0eaca044..40d249088 100644 --- a/client/coral-embed-stream/src/CommentStream.js +++ b/client/coral-embed-stream/src/CommentStream.js @@ -61,8 +61,12 @@ class CommentStream extends Component { // Set up messaging between embedded Iframe an parent component // Using recommended Pym init code which violates .eslint standards const pym = new Pym.Child({polling: 100}); - const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl); - this.props.getStream(path && path[1] || window.location); + + if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) { + this.props.getStream(pym.parentUrl); + } else { + this.props.getStream(window.location); + } } render () { diff --git a/client/coral-framework/actions/auth.js b/client/coral-framework/actions/auth.js index 7313bb30f..1058edbbb 100644 --- a/client/coral-framework/actions/auth.js +++ b/client/coral-framework/actions/auth.js @@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error}); export const checkLogin = () => dispatch => { dispatch(checkLoginRequest()); fetch(`${base}/auth`, getInit('GET')) - .then(handleResp) + .then((res) => { + if (res.status !== 200) { + throw new Error('not logged in'); + } + + return res.json(); + }) .then(user => dispatch(checkLoginSuccess(user))) .catch(error => dispatch(checkLoginFailure(error))); }; diff --git a/models/asset.js b/models/asset.js index 1092cd3e0..b5321cec3 100644 --- a/models/asset.js +++ b/models/asset.js @@ -3,7 +3,6 @@ const uuid = require('uuid'); const Schema = mongoose.Schema; const AssetSchema = new Schema({ - id: { type: String, default: uuid.v4, @@ -19,12 +18,18 @@ const AssetSchema = new Schema({ type: String, default: 'article' }, - headline: String, - summary: String, + scraped: { + type: Date, + default: null + }, + title: String, + description: String, + image: String, section: String, subsection: String, - authors: [String], - publication_date: Date + author: String, + publication_date: Date, + modified_date: Date }, { versionKey: false, timestamps: { @@ -36,80 +41,42 @@ const AssetSchema = new Schema({ /** * Search for assets. Currently only returns all. */ -AssetSchema.statics.search = function(query) { - - return Asset.find(query).exec(); - -}; +AssetSchema.statics.search = (query) => Asset.find(query); /** * Finds an asset by its id. * @param {String} id identifier of the asset (uuid). */ -AssetSchema.statics.findById = function(id) { - - return Asset.findOne({id}).exec(); - -}; +AssetSchema.statics.findById = (id) => Asset.findOne({id}); /** * Finds a asset by its url. * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findByUrl = function(url) { - - return Asset.findOne({'url': url}).exec(); - -}; +AssetSchema.statics.findByUrl = (url) => Asset.findOne({url}); /** * Finds a asset by its url. + * + * NOTE: This function has scalability concerns regarding mongoose's decision + * always write {updated_at: new Date()} on every call to findOneAndUpdate + * even though the update document exactly matches the query document... In + * the future this function should never update, only findOneAndCreate but this + * is not possible with the mongoose driver. + * * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findOrCreateByUrl = function(url) { +AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, { - return Asset.findOne({url}) - .then((asset) => asset ? asset - : Asset.upsert({url})); -}; + // Ensure that if it's new, we return the new object created. + new: true, -/** - * Upserts an asset. -*/ -AssetSchema.statics.upsert = function(data) { - // If an id is not sent, create one. - if (typeof data.id === 'undefined') { - data.id = uuid.v4(); - } + // Perform an upsert in the event that this doesn't exist. + upsert: true, - // Perform the upsert against the id field. - let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec() - .then(() => { - - // Pull the freshly minted asset out and return. - return Asset.findById(data.id); - - }) - .catch((err) => { - - console.error('Error upserting asset.', err); - //return new Promise(); // ??? what do we return on error? - - }); - - return updatePromise; - -}; - -/** - * Remove assets from the db. - * @param {String} query bson query to identify assets to be removed. -*/ -AssetSchema.statics.removeAll = function(query) { - - return Asset.remove(query).exec(); - -}; + // Set the default values if not provided based on the mongoose models. + setDefaultsOnInsert: true +}); const Asset = mongoose.model('Asset', AssetSchema); diff --git a/package.json b/package.json index 9300c347f..bc2f72f20 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,14 @@ "description": "A commenting platform from The Coral Project. https://coralproject.net", "main": "app.js", "scripts": { - "start": "./bin/www", + "start": "./bin/cli serve --jobs", "build": "NODE_ENV=production webpack --config webpack.config.js --bail", "build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch", "lint": "eslint bin/* .", "lint-fix": "eslint . --fix", - "test": "mocha --compilers js:babel-core/register --recursive tests", - "test-watch": "mocha --compilers js:babel-core/register --recursive -w tests", - "embed-start": "NODE_ENV=development npm run build && ./bin/www" + "test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests", + "test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests", + "embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs" }, "config": { "pre-git": { @@ -45,14 +45,16 @@ "express-session": "^1.14.2", "helmet": "^3.1.0", "jsonwebtoken": "^7.1.9", + "kue": "^0.11.5", "lodash": "^4.16.6", + "metascraper": "^1.0.6", "mongoose": "^4.6.5", "morgan": "^1.7.0", + "natural": "^0.4.0", "nodemailer": "^2.6.4", "passport": "^0.3.2", "passport-facebook": "^2.1.1", "passport-local": "^1.0.0", - "natural": "^0.4.0", "prompt": "^1.0.0", "react-linkify": "^0.1.3", "redis": "^2.6.3", diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js index 217c49547..18fd0b7ec 100644 --- a/routes/api/asset/index.js +++ b/routes/api/asset/index.js @@ -1,33 +1,81 @@ const express = require('express'); const router = express.Router(); -const Asset = require('../../../models/asset'); -// Search assets. +const Asset = require('../../../models/asset'); +const scraper = require('../../../services/scraper'); + +// List assets. router.get('/', (req, res, next) => { - let query = {}; + const { + limit = 20, + skip = 0, + sort = 'asc', + field = 'created_at' + } = req.query; - if (typeof req.query.url !== 'undefined') { - query.url = req.query.url; - } + // Find all the assets. + Promise.all([ + Asset + .find({}) + .sort({[field]: (sort === 'asc') ? 1 : -1}) + .skip(skip) + .limit(limit), + Asset.count() + ]) + .then(([result, count]) => { - Asset.search(query) - .then((asset) => { - res.json(asset); - }) - .catch(next); + // Send back the asset data. + res.json({ + result, + count + }); + }) + .catch((err) => { + next(err); + }); }); -// Get an asset by id -router.get('/:id', (req, res, next) => { +// Get an asset by id. +router.get('/:asset_id', (req, res, next) => { - Asset.findById(req.params.id) + // Send back the asset. + Asset + .findById(req.params.asset_id) .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + res.json(asset); }) - .catch(next); + .catch((err) => { + next(err); + }); +}); +// Adds the asset id to the queue to be scraped. +router.post('/:asset_id/scrape', (req, res, next) => { + + // Create a new asset scrape job. + Asset + .findById(req.params.asset_id) + .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + + return scraper.create(asset); + }) + .then((job) => { + + // Send the job back for monitoring. + res.status(201).json(job); + }) + .catch((err) => { + next(err); + }); }); module.exports = router; diff --git a/routes/api/auth/index.js b/routes/api/auth/index.js index b88b6fe12..369e5ec77 100644 --- a/routes/api/auth/index.js +++ b/routes/api/auth/index.js @@ -7,7 +7,18 @@ const router = express.Router(); /** * This returns the user if they are logged in. */ -router.get('/', authorization.needed(), (req, res) => { +router.get('/', (req, res, next) => { + if (req.user) { + return next(); + } + + // When there is no user on the request, then just send back a 204 to this + // request. It's not really "an error" if what they asked for isn't available, + // but it could be. + res.status(204).end(); +}, (req, res) => { + + // Send back the user object. res.json(req.user.toObject()); }); diff --git a/routes/api/index.js b/routes/api/index.js index 8c6014ed8..8da3f791b 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -3,13 +3,18 @@ const authorization = require('../../middleware/authorization'); const router = express.Router(); -router.use('/asset', require('./asset')); -router.use('/auth', require('./auth')); -router.use('/comments', authorization.needed(), require('./comments')); -router.use('/queue', require('./queue')); +router.use('/asset', authorization.needed('admin'), require('./asset')); router.use('/settings', authorization.needed('admin'), require('./settings')); -router.use('/stream', require('./stream')); -router.use('/user', require('./user')); +router.use('/queue', authorization.needed('admin'), require('./queue')); + +router.use('/comments', authorization.needed(), require('./comments')); router.use('/actions', authorization.needed(), require('./actions')); +router.use('/auth', require('./auth')); +router.use('/stream', require('./stream')); +router.use('/user', require('./user')); + +// Bind the kue handler to the /kue path. +router.use('/kue', authorization.needed('admin'), require('kue').app); + module.exports = router; diff --git a/routes/api/queue/index.js b/routes/api/queue/index.js index edc5602ef..f661992f1 100644 --- a/routes/api/queue/index.js +++ b/routes/api/queue/index.js @@ -1,7 +1,6 @@ const express = require('express'); const Comment = require('../../../models/comment'); const Setting = require('../../../models/setting'); -const authorization = require('../../../middleware/authorization'); const router = express.Router(); @@ -13,7 +12,7 @@ const router = express.Router(); // depending on the settings. The :moderation overwrites this settings. // Pre-moderation: New comments are shown in the moderator queues immediately. // Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users. -router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => { +router.get('/comments/pending', (req, res, next) => { Setting.getModerationSetting().then(function({moderation}){ Comment.moderationQueue(moderation).then((comments) => { res.status(200).json(comments); diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index a48548608..0ef97aad2 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -1,52 +1,99 @@ const express = require('express'); const _ = require('lodash'); +const scraper = require('../../../services/scraper'); const Comment = require('../../../models/comment'); const User = require('../../../models/user'); const Action = require('../../../models/action'); const Asset = require('../../../models/asset'); - const Setting = require('../../../models/setting'); const router = express.Router(); -// Find all the comments by a specific asset_url. -// . if pre: get the comments that are accepted. -// . if post: get the comments that are new and accepted. router.get('/', (req, res, next) => { // Get the asset_id for this url (or create it if it doesn't exist) Promise.all([ - Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)), + + // Find or create the asset by url. + Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)) + + // Add the found asset to the scraper if it's not already scraped. + .then((asset) => { + if (!asset.scraped) { + return scraper.create(asset).then(() => asset); + } + + return asset; + }), + + // Get the moderation setting from the settings. Setting.getModerationSetting() ]) .then(([asset, {moderation}]) => { - // Get the sitewide moderation setting and return the appropriate comments - switch(moderation){ - case 'pre': - return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]); - case 'post': - return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]); - default: - return Promise.reject(new Error('Moderation setting not found.')); + let comments; + + if (moderation === 'post') { + comments = Comment.findAcceptedByAssetId(asset.id); + } else { + + // Defaults to 'pre' moderation. + comments = Comment.findAcceptedAndNewByAssetId(asset.id); } + + return Promise.all([ + + // This is the promised component... Fetch the comments based on the + // moderation settings. + comments, + + // Send back the reference to the asset. + asset + ]); }) // Get all the users and actions for those comments. .then(([comments, asset]) => { + + // Get the user id's from the author id's as a unique array that gets + // sorted. + let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort(); + + // Fetch the users for which there is a comment available for them. + let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : []; + + // Fetch the actions for pretty much everything at this point. + let actions = Action.getActionSummaries(_.uniq([ + + // Actions can be on assets... + asset.id, + + // Comments... + ...comments.map((comment) => comment.id), + + // Or Authors... + ...userIDs + ]), req.user ? req.user.id : false); + return Promise.all([ - [asset], + + // Pass back the asset that we loaded... + asset, + + // It's comments... comments, - User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))), - Action.getActionSummaries(_.uniq([ - asset.id, - ...comments.map((comment) => comment.id), - ...comments.map((comment) => comment.author_id) - ]), req.user ? req.user.id : '') + + // All the users/authors of those comments... + users, + + // And all actions about the asset, comments, and users. + actions ]); }) - .then(([assets, comments, users, actions]) => { + .then(([asset, comments, users, actions]) => { + + // Send back the payload containing all this data. res.json({ - assets, + assets: [asset], comments, users, actions diff --git a/routes/api/user/index.js b/routes/api/user/index.js index 3251e0608..43d3b8ac0 100644 --- a/routes/api/user/index.js +++ b/routes/api/user/index.js @@ -16,7 +16,7 @@ router.get('/', authorization.needed('admin'), (req, res, next) => { page = 1, asc = 'false', limit = 50 // Total Per Page - } = req.query; + } = req.query; Promise.all([ User @@ -128,9 +128,10 @@ router.post('/request-password-reset', (req, res, next) => { return mailer.sendSimple(options); }) .then(() => { + // we want to send a 204 regardless of the user being found in the db // if we fail on missing emails, it would reveal if people are registered or not. - res.status(204).send('OK'); + res.status(204).end(); }) .catch(error => { const errorMsg = typeof error === 'string' ? error : error.message; diff --git a/routes/index.js b/routes/index.js index 0c03edcc5..a875d5127 100644 --- a/routes/index.js +++ b/routes/index.js @@ -6,11 +6,11 @@ router.use('/admin', require('./admin')); router.use('/embed', require('./embed')); router.get('/', (req, res) => { - return res.render('article', {title: 'Coral Talk'}); + res.render('article', {title: 'Coral Talk'}); }); router.get('/assets/:asset_title', (req, res) => { - return res.render('article', {title: req.params.asset_title.split('-').join(' ')}); + res.render('article', {title: req.params.asset_title.split('-').join(' ')}); }); module.exports = router; diff --git a/services/scraper.js b/services/scraper.js new file mode 100644 index 000000000..665d4fc66 --- /dev/null +++ b/services/scraper.js @@ -0,0 +1,140 @@ +const kue = require('kue'); +const queue = kue.createQueue(); +const debug = require('debug')('talk:services:scraper'); +const Asset = require('../models/asset'); +const JOB_NAME = 'scraper'; + +const metascraper = require('metascraper'); + +/** + * Exposes a service object to allow operations to execute against the scraper. + * @type {Object} + */ +const scraper = { + + /** + * creates a new scraper job and scrapes the url when it gets processed. + */ + create(asset) { + return new Promise((resolve, reject) => { + debug(`Creating job for Asset[${asset.id}]`); + + let job = queue + .create(JOB_NAME, { + title: `Scrape for asset ${asset.id}`, + asset_id: asset.id + }) + .attempts(10) + .delay(1000) + .backoff({type: 'exponential'}) + .save((err) => { + if (err) { + return reject(err); + } + + debug(`Created Job[${job.id}] for Asset[${asset.id}]`); + + return resolve(job); + }); + }); + }, + + /** + * Scrapes the given asset for metadata. + */ + scrape(asset) { + return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, { + section: ($) => $('meta[property="article:section"]').attr('content'), + modified: ($) => $('meta[property="article:modified"]').attr('content') + })); + }, + + update(id, meta) { + return Asset.update({id}, { + $set: { + title: meta.title || '', + description: meta.description || '', + image: meta.image ? meta.image : '', + author: meta.author || '', + publication_date: meta.date || '', + modified_date: meta.modified || '', + section: meta.section || '', + scraped: new Date() + } + }); + }, + + /** + * Start the queue processor for the scraper job. + */ + process() { + + debug(`Now processing ${JOB_NAME} jobs`); + + // Process jobs with the processJob function. + queue.process(JOB_NAME, (job, done) => { + + debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + Asset + + // Find the asset, or complain that it doesn't exist. + .findById(job.data.asset_id) + .then((asset) => { + if (!asset) { + throw new Error('asset not found'); + } + + return asset; + }) + + // Scrape the metadata from the asset. + .then(scraper.scrape) + + // Assign the metadata retrieved for the asset to the db. + .then((meta) => { + debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + return scraper.update(job.data.asset_id, meta); + }) + + // Finish the job because we just handled our scraping + updating the + // asset in the database. + .then(() => { + debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`); + done(); + }) + + // Handle errors that occur. + .catch((err) => { + console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err); + + done(err); + }); + }); + }, + + /** + * Shuts down the current queue to ensure that the application can shutdown + * cleanly. + */ + shutdown() { + return new Promise((resolve, reject) => { + + // Shutdown and give the queue 5 seconds to shutdown before we start + // killing jobs. + queue.shutdown(5000, (err) => { + if (err) { + return reject(err); + } + + debug(`Processing for ${JOB_NAME} jobs stopped`); + + resolve(); + }); + }); + } + +}; + +module.exports = scraper; diff --git a/swagger.yaml b/swagger.yaml index fd3b0012b..09adb97d6 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -256,6 +256,86 @@ paths: description: An error occured. schema: $ref: '#/definitions/Error' + /asset: + get: + parameters: + - name: limit + in: query + type: number + format: int32 + description: Limit the listing results + - name: skip + in: query + type: number + format: int32 + description: Skip the listing results + - name: sort + in: query + enum: + - asc + - desc + type: string + description: Sorting method + - name: field + in: query + type: string + description: Field to sort by. + responses: + 200: + description: Assets listed. + schema: + type: object + properties: + count: + type: number + description: Total number of assets found. + result: + type: array + items: + $ref: '#/definitions/Asset' + + /asset/{asset_id}: + get: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 200: + description: The requested asset. + schema: + $ref: '#/definitions/Asset' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + + /asset/{asset_id}/scrape: + post: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 201: + description: The job that was created. + schema: + $ref: '#/definitions/Job' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + /stream: get: tags: @@ -420,3 +500,10 @@ definitions: type: object Settings: type: object + Job: + type: object + properties: + id: + format: number + state: + format: string diff --git a/tests/index.js b/tests/index.js deleted file mode 100644 index 0c3d947ce..000000000 --- a/tests/index.js +++ /dev/null @@ -1,9 +0,0 @@ -const expect = require('chai').expect; - -describe('Comment', () => { - describe('#add', () => { - it('should add a comment', () => { - expect(0).to.be.equal(0); - }); - }); -}); diff --git a/tests/models/action.js b/tests/models/action.js index 87d53b7da..c354b8f83 100644 --- a/tests/models/action.js +++ b/tests/models/action.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Action = require('../../models/action'); const expect = require('chai').expect; diff --git a/tests/models/asset.js b/tests/models/asset.js index d0c3d3429..a5c4ae62b 100644 --- a/tests/models/asset.js +++ b/tests/models/asset.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Asset = require('../../models/asset'); const expect = require('chai').expect; @@ -74,35 +70,4 @@ describe('Asset: model', () => { }); }); }); - - describe('#upsert', ()=> { - it('should insert an asset with no id', () => { - return Asset.upsert({url: 'http://newasset.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id'); - }); - }); - - it('should update an asset when the id exists', () => { - return Asset.upsert({id: 1, url: 'http://new.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id') - .and.to.equal('1'); - expect(asset).to.have.property('url') - .and.to.equal('http://new.test.com'); - }); - }); - }); - - describe('#removeAll', ()=> { - it('should insert an asset with no id', () => { - return Asset.removeAll({id:1}) - .then(() => { - return Asset.findById(1); - }) - .then((result) => { - expect(result).to.be.null; - }); - }); - }); }); diff --git a/tests/models/comment.js b/tests/models/comment.js index f8dd7f9f5..07834a186 100644 --- a/tests/models/comment.js +++ b/tests/models/comment.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Comment = require('../../models/comment'); const User = require('../../models/user'); const Action = require('../../models/action'); diff --git a/tests/models/setting.js b/tests/models/setting.js index 3f4dc27fa..a36b363fb 100644 --- a/tests/models/setting.js +++ b/tests/models/setting.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Setting = require('../../models/setting'); const expect = require('chai').expect; diff --git a/tests/models/user.js b/tests/models/user.js index f77543b4b..c8af92054 100644 --- a/tests/models/user.js +++ b/tests/models/user.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const User = require('../../models/user'); const expect = require('chai').expect; diff --git a/tests/utils/mongoose.js b/tests/mongoose.js similarity index 71% rename from tests/utils/mongoose.js rename to tests/mongoose.js index 1549440a6..9d3ba1195 100644 --- a/tests/utils/mongoose.js +++ b/tests/mongoose.js @@ -1,8 +1,4 @@ -const mongoose = require('../../mongoose'); - -// Ensure the NODE_ENV is set to 'test', -// this is helpful when you would like to change behavior when testing. -process.env.NODE_ENV = 'test'; +const mongoose = require('../mongoose'); beforeEach(function (done) { function clearDB() { diff --git a/tests/utils/passport.js b/tests/passport.js similarity index 90% rename from tests/utils/passport.js rename to tests/passport.js index 401e50b77..2d1c53ab9 100644 --- a/tests/utils/passport.js +++ b/tests/passport.js @@ -1,4 +1,4 @@ -const authorization = require('../../middleware/authorization'); +const authorization = require('../middleware/authorization'); // Add the passport middleware here before it's setup. authorization.middleware.push((req, res, next) => { diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js index 542e88c42..fb67b7b48 100644 --- a/tests/routes/api/assets/index.js +++ b/tests/routes/api/assets/index.js @@ -1,36 +1,10 @@ -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +describe('/assets', () => { -const chai = require('chai'); -const server = require('../../../../app'); + describe('GET', () => { -// Setup chai. -chai.should(); -chai.use(require('chai-http')); + it('should return assets that we search for'); + it('should not return assets that we do not search for'); -describe('Asset: routes', () => { - - describe('/GET Asset', () => { - describe('#get', () => { - it('It should get an empty array when there are no assets.', (done) => { - - chai.request(server) - .get('/api/v1/asset') - .set(passport.inject({roles: ['admin']})) - .end((err, res) => { - - if (err) { - throw new Error(err); - } - - res.should.have.status(200); - res.body.should.be.a('array'); - res.body.length.should.be.eql(0); - done(); - }); - - }); - }); }); }); diff --git a/tests/routes/api/auth/index.js b/tests/routes/api/auth/index.js index dd49a1472..ddca63fe8 100644 --- a/tests/routes/api/auth/index.js +++ b/tests/routes/api/auth/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index 29775d3d0..ff7f3a9d4 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/queue/index.js b/tests/routes/api/queue/index.js index 4eff3d384..74137a49a 100644 --- a/tests/routes/api/queue/index.js +++ b/tests/routes/api/queue/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/settings/index.js b/tests/routes/api/settings/index.js index 86feca1eb..9f4466a7f 100644 --- a/tests/routes/api/settings/index.js +++ b/tests/routes/api/settings/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/stream/index.js b/tests/routes/api/stream/index.js index 009184b6b..7d05c9708 100644 --- a/tests/routes/api/stream/index.js +++ b/tests/routes/api/stream/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; @@ -92,8 +90,8 @@ describe('api/stream: routes', () => { .then(res => { expect(res).to.have.status(200); expect(res.body.assets.length).to.equal(1); - expect(res.body.comments.length).to.equal(1); - expect(res.body.users.length).to.equal(1); + expect(res.body.comments.length).to.equal(2); + expect(res.body.users.length).to.equal(2); expect(res.body.actions.length).to.equal(1); }); }); diff --git a/tests/services/scraper.js b/tests/services/scraper.js new file mode 100644 index 000000000..f3b5d006c --- /dev/null +++ b/tests/services/scraper.js @@ -0,0 +1,22 @@ +describe('scraper: services', () => { + describe('#create', () => { + it('should create a new kue job'); + }); + + describe('#scrape', () => { + it('should scrape complete information'); + it('should scrape what it can'); + }); + + describe('#update', () => { + it('should update the database record entries from the meta'); + }); + + describe('#process', () => { + it('should start the processor to scrape assets'); + }); + + describe('#shutdown', () => { + it('should shutdown the job processor'); + }); +}); diff --git a/util.js b/util.js new file mode 100644 index 000000000..6907d90cd --- /dev/null +++ b/util.js @@ -0,0 +1,42 @@ +const util = module.exports = {}; + +/** + * Stores an array of functions that should be executed in the event that the + * application needs to shutdown. + * @type {Array} + */ +util.toshutdown = []; + +/** + * Calls all the shutdown functions and then ends the process. + * @param {Number} [defaultCode=0] default return code upon sucesfull shutdown. + */ +util.shutdown = (defaultCode = 0) => { + Promise + .all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func)) + .then(() => { + process.exit(defaultCode); + }) + .catch((err) => { + console.error(err); + + process.exit(1); + }); +}; + +/** + * Waits until an event is triggered by the node runtime and elevates a series + * of jobs to be ran in the event we need to shutdown. + * @param {Array} jobs Array of promise capable shutdown functions that are + * executed. + */ +util.onshutdown = (jobs) => { + + // Add the new jobs to shutdown to the object reference. + util.toshutdown = util.toshutdown.concat(jobs); +}; + +// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the +// event that we have an external event. +process.on('SIGTERM', () => util.shutdown()); +process.on('SIGINT', () => util.shutdown()); diff --git a/views/article.ejs b/views/article.ejs index 7d168f960..71db59593 100644 --- a/views/article.ejs +++ b/views/article.ejs @@ -1,8 +1,10 @@ - + + +