diff --git a/.eslintrc.json b/.eslintrc.json index 6ac5a08e6..035a86189 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,25 +5,15 @@ }, "extends": "eslint:recommended", "rules": { - "indent": [ - "error", + "indent": ["error", 2 ], "no-console": [ 0 ], - "linebreak-style": [ - "error", - "unix" - ], - "quotes": [ - "error", - "single" - ], - "semi": [ - "error", - "always" - ], + "linebreak-style": ["error", "unix"], + "quotes": ["error", "single"], + "semi": ["error", "always"], "no-template-curly-in-string": [1], "no-unsafe-negation": [1], "array-callback-return": [1], @@ -35,7 +25,6 @@ "no-throw-literal": [2], "yoda": [1], "no-path-concat": [2], - "no-process-exit": [2], "eol-last": [1], "no-continue": [1], "no-nested-ternary": [1], @@ -46,20 +35,20 @@ "no-const-assign": [2], "no-duplicate-imports": [2], "prefer-template": [1], - "comma-spacing": [ - "error", - { + "comma-spacing": ["error", { "after": true - } - ], + }], "no-var": [2], "no-lonely-if": [2], "curly": [2], - "no-unused-vars": ["error", { "argsIgnorePattern": "next" }], - "no-multiple-empty-lines": [ - "error", - {"max": 1} - ], - "newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }] + "no-unused-vars": ["error", { + "argsIgnorePattern": "next" + }], + "no-multiple-empty-lines": ["error", { + "max": 1 + }], + "newline-per-chained-call": ["error", { + "ignoreChainWithDepth": 2 + }] } } diff --git a/app.js b/app.js index 8008cb01e..e84eb4b79 100644 --- a/app.js +++ b/app.js @@ -45,7 +45,7 @@ const session_opts = { }, store: new RedisStore({ ttl: 1800, - client: redis, + client: redis.createClient(), }) }; diff --git a/bin/cli b/bin/cli index bec5ea84f..d8148282f 100755 --- a/bin/cli +++ b/bin/cli @@ -19,7 +19,10 @@ const pkg = require('../package.json'); program .version(pkg.version) + .command('serve', 'serve the application') + .command('assets', 'interact with assets') .command('settings', 'work with the application settings') + .command('jobs', 'work with the job queues') .command('users', 'work with the application auth') .parse(process.argv); diff --git a/bin/cli-assets b/bin/cli-assets new file mode 100755 index 000000000..99965c6d9 --- /dev/null +++ b/bin/cli-assets @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const pkg = require('../package.json'); +const Table = require('cli-table'); +const Asset = require('../models/asset'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +// Register the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); + +/** + * Lists all the assets registered in the database. + */ +function listAssets() { + Asset + .find({}) + .sort({'created_at': 1}) + .then((asset) => { + let table = new Table({ + head: [ + 'ID', + 'Title', + 'URL' + ] + }); + + asset.forEach((asset) => { + table.push([ + asset.id, + asset.title ? asset.title : '', + asset.url ? asset.url : '' + ]); + }); + + console.log(table.toString()); + util.shutdown(); + }) + .catch((err) => { + console.error(err); + util.shutdown(1); + }); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .version(pkg.version); + +program + .command('list') + .description('list all the assets in the database') + .action(listAssets); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (!process.argv.slice(2).length) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-jobs b/bin/cli-jobs new file mode 100755 index 000000000..f14276d38 --- /dev/null +++ b/bin/cli-jobs @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +/** + * Module dependencies. + */ + +const program = require('commander'); +const scraper = require('../services/scraper'); +const util = require('../util'); +const mongoose = require('../mongoose'); + +util.onshutdown([ + () => mongoose.disconnect() +]); + +function processJobs() { + + // Start the processor. + scraper.process(); + + // The scraper only needs to shutdown when the scraper has actually been + // started. + util.onshutdown([ + () => scraper.shutdown() + ]); +} + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .command('process') + .description('starts job processing') + .action(processJobs); + +program.parse(process.argv); + +// If there is no command listed, output help. +if (process.argv.length <= 2) { + program.outputHelp(); + util.shutdown(); +} diff --git a/bin/cli-serve b/bin/cli-serve new file mode 100755 index 000000000..14c1261d7 --- /dev/null +++ b/bin/cli-serve @@ -0,0 +1,132 @@ +#!/usr/bin/env node + +/** + * Setup the debug paramater. + */ + +process.env.DEBUG = process.env.TALK_DEBUG; + +const app = require('../app'); +const debug = require('debug')('talk:server'); +const http = require('http'); +const init = require('../init'); +const scraper = require('../services/scraper'); +const mongoose = require('../mongoose'); +const util = require('../util'); + +/** +* Get port from environment and store in Express. +*/ +const port = normalizePort(process.env.TALK_PORT || '3000'); + +app.set('port', port); + +/** +* Create HTTP server. +*/ +const server = http.createServer(app); + +/** + * Event listener for HTTP server "error" event. + */ +function onError(error) { + if (error.syscall !== 'listen') { + throw error; + } + + let bind = typeof port === 'string' + ? `Pipe ${port}` + : `Port ${port}`; + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + console.error(`${bind} requires elevated privileges`); + break; + case 'EADDRINUSE': + console.error(`${bind} is already in use`); + break; + } + + throw error; +} + +/** + * Normalize a port into a number, string, or false. + */ + +function normalizePort(val) { + let port = parseInt(val, 10); + + if (isNaN(port)) { + // named pipe + return val; + } + + if (port >= 0) { + // port number + return port; + } + + return false; +} + +/** + * Event listener for HTTP server "listening" event. + */ + +function onListening() { + let addr = server.address(); + let bind = typeof addr === 'string' + ? `pipe ${ addr}` + : `port ${ addr.port}`; + debug(`Listening on ${ bind}`); +} + +/** + * Start the app. + */ +function startApp() { + init().then(() => { + + /** + * Listen on provided port, on all network interfaces. + */ + server.listen(port); + server.on('error', onError); + server.on('listening', onListening); + }); +} + +/** + * Module dependencies. + */ + +const program = require('commander'); + +//============================================================================== +// Setting up the program command line arguments. +//============================================================================== + +program + .option('-j, --jobs', 'enable job processing on this thread') + .parse(process.argv); + +// Start the application serving. +startApp(); + +// Enable job processing on the thread if enabled. +if (program.jobs) { + + // Start the processor. + scraper.process(); +} + +// Define a safe shutdown function to call in the event we need to shutdown +// because the node hooks are below which will interrupt the shutdown process. +// Shutdown the mongoose connection, the app server, and the scraper. +util.onshutdown([ + () => program.jobs ? scraper.shutdown() : null, + () => mongoose.disconnect(), + () => server.close() +]); diff --git a/bin/cli-settings b/bin/cli-settings index cff6c04ed..c639a5ca5 100755 --- a/bin/cli-settings +++ b/bin/cli-settings @@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG; */ const program = require('commander'); +const mongoose = require('../mongoose'); +const Setting = require('../models/setting'); +const util = require('../util'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); //============================================================================== // Setting up the program command line arguments. @@ -20,19 +28,17 @@ program .command('init') .description('initilizes the talk settings') .action(() => { - const mongoose = require('../mongoose'); - const Setting = require('../models/setting'); const defaults = {id: '1', moderation: 'pre'}; Setting .update({id: '1'}, {$setOnInsert: defaults}, {upsert: true}) .then(() => { console.log('Created settings object.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(`failed to create the settings object ${JSON.stringify(err)}`); - throw new Error(err); // just to be safe + util.shutdown(1); }); }); @@ -41,4 +47,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/cli-users b/bin/cli-users index 54d7f2927..2e0dc84e6 100755 --- a/bin/cli-users +++ b/bin/cli-users @@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG; const program = require('commander'); const pkg = require('../package.json'); const prompt = require('prompt'); +const User = require('../models/user'); +const mongoose = require('../mongoose'); +const util = require('../util'); +const Table = require('cli-table'); + +// Regeister the shutdown criteria. +util.onshutdown([ + () => mongoose.disconnect() +]); /** * Prompts for input and registers a user based on those. */ function createUser(options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - return new Promise((resolve, reject) => { if (options.flag_mode) { @@ -74,11 +80,11 @@ function createUser(options) { }) .then((user) => { console.log(`Created user ${user.id}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -86,20 +92,17 @@ function createUser(options) { * Deletes a user. */ function deleteUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .findOneAndRemove({ id: userID }) .then(() => { console.log('Deleted user'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(); }); } @@ -107,9 +110,6 @@ function deleteUser(userID) { * Changes the password for a user. */ function passwd(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - prompt.start(); prompt.get([ @@ -128,13 +128,13 @@ function passwd(userID) { ], (err, result) => { if (err) { console.error(err); - mongoose.disconnect(); + util.shutdown(); return; } if (result.password !== result.confirmPassword) { console.error(new Error('Password mismatch')); - mongoose.disconnect(); + util.shutdown(1); return; } @@ -142,11 +142,11 @@ function passwd(userID) { .changePassword(userID, result.password) .then(() => { console.log('Password changed.'); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); }); } @@ -155,9 +155,6 @@ function passwd(userID) { * Updates the user from the options array. */ function updateUser(userID, options) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - const updates = []; if (options.email && typeof options.email === 'string' && options.email.length > 0) { @@ -189,11 +186,11 @@ function updateUser(userID, options) { .all(updates.map((q) => q.exec())) .then(() => { console.log(`User ${userID} updated.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -201,10 +198,6 @@ function updateUser(userID, options) { * Lists all the users registered in the database. */ function listUsers() { - const Table = require('cli-table'); - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .all() .then((users) => { @@ -229,11 +222,11 @@ function listUsers() { }); console.log(table.toString()); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -243,18 +236,15 @@ function listUsers() { * @param {String} srcUserID id of the user to which is the source of the merge */ function mergeUsers(dstUserID, srcUserID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .mergeUsers(dstUserID, srcUserID) .then(() => { console.log(`User ${srcUserID} was merged into user ${dstUserID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -264,18 +254,15 @@ function mergeUsers(dstUserID, srcUserID) { * @param {String} role the role to add */ function addRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .addRoleToUser(userID, role) .then(() => { console.log(`Added the ${role} role to User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -285,18 +272,15 @@ function addRole(userID, role) { * @param {String} role the role to remove */ function removeRole(userID, role) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .removeRoleFromUser(userID, role) .then(() => { console.log(`Removed the ${role} role from User ${userID}.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -305,18 +289,15 @@ function removeRole(userID, role) { * @param {String} userID the ID of a user to disable */ function disableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .disableUser(userID) .then(() => { console.log(`User ${userID} was disabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -325,18 +306,15 @@ function disableUser(userID) { * @param {String} userID the ID of a user to enable */ function enableUser(userID) { - const User = require('../models/user'); - const mongoose = require('../mongoose'); - User .enableUser(userID) .then(() => { console.log(`User ${userID} was enabled.`); - mongoose.disconnect(); + util.shutdown(); }) .catch((err) => { console.error(err); - mongoose.disconnect(); + util.shutdown(1); }); } @@ -408,4 +386,5 @@ program.parse(process.argv); // If there is no command listed, output help. if (!process.argv.slice(2).length) { program.outputHelp(); + util.shutdown(); } diff --git a/bin/www b/bin/www deleted file mode 100755 index 3e9e20918..000000000 --- a/bin/www +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env node - -/** - * Setup the debug paramater. - */ - -process.env.DEBUG = process.env.TALK_DEBUG; - -/** - * Module dependencies. - */ - -const app = require('../app'); -const debug = require('debug')('talk:server'); -const http = require('http'); -const init = require('../init'); -const port = normalizePort(process.env.TALK_PORT || '3000'); - -let server; - -init().then(() => { - - /** - * Get port from environment and store in Express. - */ - app.set('port', port); - - /** - * Create HTTP server. - */ - server = http.createServer(app); - - /** - * Listen on provided port, on all network interfaces. - */ - server.listen(port); - server.on('error', onError); - server.on('listening', onListening); -}); - -/** - * Normalize a port into a number, string, or false. - */ - -function normalizePort(val) { - let port = parseInt(val, 10); - - if (isNaN(port)) { - // named pipe - return val; - } - - if (port >= 0) { - // port number - return port; - } - - return false; -} - -/** - * Event listener for HTTP server "error" event. - */ - -function onError(error) { - if (error.syscall !== 'listen') { - throw error; - } - - let bind = typeof port === 'string' - ? `Pipe ${ port}` - : `Port ${ port}`; - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - console.error(`${bind} requires elevated privileges`); - break; - case 'EADDRINUSE': - console.error(`${bind} is already in use`); - break; - } - - throw error; -} - -/** - * Event listener for HTTP server "listening" event. - */ - -function onListening() { - let addr = server.address(); - let bind = typeof addr === 'string' - ? `pipe ${ addr}` - : `port ${ addr.port}`; - debug(`Listening on ${ bind}`); -} diff --git a/cache.js b/cache.js index efe689f9c..9345f8cde 100644 --- a/cache.js +++ b/cache.js @@ -1,6 +1,8 @@ const redis = require('./redis'); -const cache = module.exports = {}; +const cache = module.exports = { + client: redis.createClient() +}; /** * This collects a key that may either be an array or a string and creates a @@ -51,7 +53,7 @@ cache.wrap = (key, expiry, work) => { * @return {Promise} */ cache.get = (key) => new Promise((resolve, reject) => { - redis.get(keyfunc(key), (err, reply) => { + cache.client.get(keyfunc(key), (err, reply) => { if (err) { return reject(err); } @@ -87,7 +89,7 @@ cache.set = (key, value, expiry) => new Promise((resolve, reject) => { // Serialize the value as JSON. let reply = JSON.stringify(value); - redis.set(keyfunc(key), reply, 'EX', expiry, (err) => { + cache.client.set(keyfunc(key), reply, 'EX', expiry, (err) => { if (err) { return reject(err); } diff --git a/client/coral-admin/src/actions/auth.js b/client/coral-admin/src/actions/auth.js index c5a03132f..2f8f1041e 100644 --- a/client/coral-admin/src/actions/auth.js +++ b/client/coral-admin/src/actions/auth.js @@ -1,5 +1,5 @@ import * as actions from '../constants/auth'; -import {base, handleResp, getInit} from '../helpers/response'; +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; // Check Login @@ -17,3 +17,17 @@ export const checkLogin = () => dispatch => { }) .catch(error => dispatch(checkLoginFailure(error))); }; + +// LogOut Actions + +const logOutRequest = () => ({type: actions.LOGOUT_REQUEST}); +const logOutSuccess = () => ({type: actions.LOGOUT_SUCCESS}); +const logOutFailure = () => ({type: actions.LOGOUT_FAILURE}); + +export const logout = () => dispatch => { + dispatch(logOutRequest()); + fetch(`${base}/auth`, getInit('DELETE')) + .then(handleResp) + .then(() => dispatch(logOutSuccess())) + .catch(error => dispatch(logOutFailure(error))); +}; diff --git a/client/coral-admin/src/actions/community.js b/client/coral-admin/src/actions/community.js index 7a4112f8b..5921573d1 100644 --- a/client/coral-admin/src/actions/community.js +++ b/client/coral-admin/src/actions/community.js @@ -9,7 +9,7 @@ import { SET_ROLE } from '../constants/community'; -import {base, getInit, handleResp} from '../helpers/response'; +import {base, getInit, handleResp} from '../../../coral-framework/helpers/response'; export const fetchCommenters = (query = {}) => dispatch => { dispatch(requestFetchCommenters()); diff --git a/client/coral-admin/src/actions/settings.js b/client/coral-admin/src/actions/settings.js index f71730663..6a133ddb5 100644 --- a/client/coral-admin/src/actions/settings.js +++ b/client/coral-admin/src/actions/settings.js @@ -1,3 +1,5 @@ +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; + export const SETTINGS_LOADING = 'SETTINGS_LOADING'; export const SETTINGS_RECEIVED = 'SETTINGS_RECEIVED'; export const SETTINGS_FETCH_ERROR = 'SETTINGS_FETCH_ERROR'; @@ -8,34 +10,6 @@ export const SAVE_SETTINGS_LOADING = 'SAVE_SETTINGS_LOADING'; export const SAVE_SETTINGS_SUCCESS = 'SAVE_SETTINGS_SUCCESS'; export const SAVE_SETTINGS_FAILED = 'SAVE_SETTINGS_FAILED'; -const base = '/api/v1'; - -const getInit = (method, body) => { - const headers = new Headers({ - 'Content-Type': 'application/json', - 'Accept': 'application/json' - }); - - const init = {method, headers}; - if (method.toLowerCase() !== 'get') { - init.body = JSON.stringify(body); - } - - return init; -}; - -const handleResp = res => { - if (res.status === 401) { - throw new Error('Not Authorized to make this request'); - } else if (res.status > 399) { - throw new Error('Error! Status ', res.status); - } else if (res.status === 204) { - return res.text(); - } else { - return res.json(); - } -}; - export const fetchSettings = () => dispatch => { dispatch({type: SETTINGS_LOADING}); fetch(`${base}/settings`, getInit('GET')) diff --git a/client/coral-admin/src/components/FullLoading.css b/client/coral-admin/src/components/FullLoading.css new file mode 100644 index 000000000..8d850d381 --- /dev/null +++ b/client/coral-admin/src/components/FullLoading.css @@ -0,0 +1,12 @@ +.layout { + max-width: 800px; + margin: 0 auto; +} + +.layout h1 { + font-size: 40px; +} + +.layout img { + width: 100%; +} diff --git a/client/coral-admin/src/components/FullLoading.js b/client/coral-admin/src/components/FullLoading.js new file mode 100644 index 000000000..dee584aed --- /dev/null +++ b/client/coral-admin/src/components/FullLoading.js @@ -0,0 +1,13 @@ +import React from 'react'; +import {Layout} from 'react-mdl'; +import styles from './FullLoading.css'; +import {CoralLogo} from 'coral-ui'; + +export const FullLoading = () => ( + +
+

Loading

+ +
+ +); diff --git a/client/coral-admin/src/components/ui/Header.css b/client/coral-admin/src/components/ui/Header.css index 3d4e7dc77..a0d7dd30d 100644 --- a/client/coral-admin/src/components/ui/Header.css +++ b/client/coral-admin/src/components/ui/Header.css @@ -1,6 +1,5 @@ .header { background: #505050; - overflow: hidden; } .header > div { @@ -14,8 +13,35 @@ background: #232323; } -.version { +.rightPanel { position: absolute; right: 0; - width: 50px; + width: 170px; +} + +.rightPanel ul { + list-style: none; + line-height: 38px; +} + +.rightPanel li { + display: inline-block; + float: right; + margin-left: 15px; +} + +.rightPanel .settings { + vertical-align: middle; + border-radius: 3px; + border: solid 1px #9e9e9e; + line-height: 10px; +} + +.rightPanel .settings > div { + position: relative; +} + +.rightPanel .settings:hover { + background: rgba(158, 158, 158, 0.69); + cursor: pointer; } diff --git a/client/coral-admin/src/components/ui/Header.js b/client/coral-admin/src/components/ui/Header.js index 7ba88d25a..e4d151d30 100644 --- a/client/coral-admin/src/components/ui/Header.js +++ b/client/coral-admin/src/components/ui/Header.js @@ -1,21 +1,36 @@ import React from 'react'; -import {Navigation, Header} from 'react-mdl'; +import {Navigation, Header, IconButton, MenuItem, Menu} from 'react-mdl'; import {Link, IndexLink} from 'react-router'; import styles from './Header.css'; import I18n from 'coral-framework/modules/i18n/i18n'; import translations from '../../translations.json'; import {Logo} from './Logo'; -export default () => ( +export default ({handleLogout}) => (
- {lang.t('configure.moderate')} - {lang.t('configure.community')} - {lang.t('configure.configure')} + {lang.t('configure.moderate')} + {lang.t('configure.community')} + {lang.t('configure.configure')} -
- {`v${process.env.VERSION}`} +
+
    +
  • +
    + + + Sign Out + +
    +
    • +
  • + {`v${process.env.VERSION}`} +
    • +
); diff --git a/client/coral-admin/src/components/ui/Layout.js b/client/coral-admin/src/components/ui/Layout.js index 3e1b9cf2d..46c7aa7fa 100644 --- a/client/coral-admin/src/components/ui/Layout.js +++ b/client/coral-admin/src/components/ui/Layout.js @@ -4,9 +4,9 @@ import Header from './Header'; import Drawer from './Drawer'; import styles from './Layout.css'; -export const Layout = ({children}) => ( +export const Layout = ({children, ...props}) => ( -
+
{children} diff --git a/client/coral-admin/src/components/ui/Logo.css b/client/coral-admin/src/components/ui/Logo.css index e764af627..f89bf3d5d 100644 --- a/client/coral-admin/src/components/ui/Logo.css +++ b/client/coral-admin/src/components/ui/Logo.css @@ -1,7 +1,9 @@ .logo h1 { color: #272727; font-size: 20px; - padding: 0 30px; + margin: 0; + line-height: 60px; + padding: 0 20px; } .logo span { @@ -13,6 +15,7 @@ .logo { background: #E5E5E5; + height: 100%; } diff --git a/client/coral-admin/src/containers/LayoutContainer.js b/client/coral-admin/src/containers/LayoutContainer.js index 5f3cb0cff..f263c33f5 100644 --- a/client/coral-admin/src/containers/LayoutContainer.js +++ b/client/coral-admin/src/containers/LayoutContainer.js @@ -1,37 +1,31 @@ import React, {Component} from 'react'; import {connect} from 'react-redux'; import {Layout} from '../components/ui/Layout'; -import {checkLogin} from '../actions/auth'; -import {NotFound} from '../components/NotFound'; +import {checkLogin, logout} from '../actions/auth'; +import {FullLoading} from '../components/FullLoading'; import {PermissionRequired} from '../components/PermissionRequired'; class LayoutContainer extends Component { componentWillMount () { - this.props.checkLogin(); + const {checkLogin} = this.props; + checkLogin(); } render () { - const {isAdmin, loggedIn} = this.props.auth; - - if (!loggedIn) { - return ; - } - - if (!isAdmin && loggedIn) { - return ; - } - - return ; + const {isAdmin, loggedIn, loadingUser} = this.props.auth; + if (loadingUser) { return ; } + if (!isAdmin) { return ; } + if (isAdmin && loggedIn) { return ; } + return ; } } -LayoutContainer.propTypes = {}; - const mapStateToProps = state => ({ auth: state.auth.toJS() }); const mapDispatchToProps = dispatch => ({ checkLogin: () => dispatch(checkLogin()), + handleLogout: () => dispatch(logout()) }); export default connect( diff --git a/client/coral-admin/src/reducers/auth.js b/client/coral-admin/src/reducers/auth.js index 59dccac5e..f897c1bae 100644 --- a/client/coral-admin/src/reducers/auth.js +++ b/client/coral-admin/src/reducers/auth.js @@ -9,19 +9,25 @@ const initialState = Map({ export default function auth (state = initialState, action) { switch (action.type) { + case actions.CHECK_LOGIN_REQUEST: + return state + .set('loadingUser', true); case actions.CHECK_LOGIN_FAILURE: return state .set('loggedIn', false) + .set('loadingUser', false) .set('user', null); case actions.CHECK_LOGIN_SUCCESS: return state .set('loggedIn', true) + .set('loadingUser', false) .set('isAdmin', action.isAdmin) .set('user', action.user); case actions.LOGOUT_SUCCESS: return state .set('loggedIn', false) - .set('user', null); + .set('user', null) + .set('isAdmin', false); default : return state; } diff --git a/client/coral-admin/src/services/talk-adapter.js b/client/coral-admin/src/services/talk-adapter.js index 361a6479e..6b872d12d 100644 --- a/client/coral-admin/src/services/talk-adapter.js +++ b/client/coral-admin/src/services/talk-adapter.js @@ -1,3 +1,4 @@ +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; /** * The adapter is a redux middleware that interecepts the actions that need @@ -7,9 +8,6 @@ * for the coral but also for wordpress comments, disqus and many more. */ -// Default headers for json payloads. -const jsonHeader = new Headers({'Content-Type': 'application/json'}); - // Intercept redux actions and act over the ones we are interested export default store => next => action => { @@ -35,11 +33,11 @@ export default store => next => action => { const fetchModerationQueueComments = store => Promise.all([ - fetch('/api/v1/queue/comments/pending'), - fetch('/api/v1/comments?status=rejected'), - fetch('/api/v1/comments?action_type=flag') + fetch(`${base}/queue/comments/pending`, getInit('GET')), + fetch(`${base}/comments?status=rejected`, getInit('GET')), + fetch(`${base}/comments?action_type=flag`, getInit('GET')) ]) -.then(res => Promise.all(res.map(r => r.json()))) +.then(res => Promise.all(res.map(handleResp))) .then(res => { res[2] = res[2].map(comment => { comment.flagged = true; return comment; }); return res.reduce((prev, curr) => prev.concat(curr), []); @@ -51,26 +49,22 @@ Promise.all([ // Update a comment. Now to update a comment we need to send back the whole object const updateComment = (store, comment) => { - fetch(`/api/v1/comments/${comment.get('id')}/status`, { - method: 'PUT', - headers: jsonHeader, - body: JSON.stringify({status: comment.get('status')}) - }) - .then(res => res.json()) + fetch(`${base}/comments/${comment.get('id')}/status`, getInit('PUT', {status: comment.get('status')})) + .then(handleResp) .then(res => store.dispatch({type: 'COMMENT_UPDATE_SUCCESS', res})) .catch(error => store.dispatch({type: 'COMMENT_UPDATE_FAILED', error})); }; // Create a new comment -const createComment = (store, name, comment) => -fetch('/api/v1/comments', { - method: 'POST', - body: JSON.stringify({ +const createComment = (store, name, comment) => { + const body = { status: 'Untouched', body: comment, name: name, createdAt: Date.now() - }) -}).then(res => res.json()) -.then(res => store.dispatch({type: 'COMMENT_CREATE_SUCCESS', comment: res})) -.catch(error => store.dispatch({type: 'COMMENT_CREATE_FAILED', error})); + }; + return fetch(`${base}/comments`, getInit('POST', body)) + .then(handleResp) + .then(res => store.dispatch({type: 'COMMENT_CREATE_SUCCESS', comment: res})) + .catch(error => store.dispatch({type: 'COMMENT_CREATE_FAILED', error})); +}; diff --git a/client/coral-embed-stream/src/CommentStream.js b/client/coral-embed-stream/src/CommentStream.js index a0eaca044..40d249088 100644 --- a/client/coral-embed-stream/src/CommentStream.js +++ b/client/coral-embed-stream/src/CommentStream.js @@ -61,8 +61,12 @@ class CommentStream extends Component { // Set up messaging between embedded Iframe an parent component // Using recommended Pym init code which violates .eslint standards const pym = new Pym.Child({polling: 100}); - const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl); - this.props.getStream(path && path[1] || window.location); + + if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) { + this.props.getStream(pym.parentUrl); + } else { + this.props.getStream(window.location); + } } render () { diff --git a/client/coral-embed-stream/src/index.js b/client/coral-embed-stream/src/index.js index f2a584ed1..de14edbc7 100644 --- a/client/coral-embed-stream/src/index.js +++ b/client/coral-embed-stream/src/index.js @@ -2,9 +2,7 @@ import React from 'react'; import {render} from 'react-dom'; import CommentStream from './CommentStream'; import {Provider} from 'react-redux'; -import {fetchConfig, store} from '../../coral-framework'; - -store.dispatch(fetchConfig()); +import {store} from '../../coral-framework'; render( diff --git a/client/coral-framework/actions/auth.js b/client/coral-framework/actions/auth.js index 7313bb30f..1058edbbb 100644 --- a/client/coral-framework/actions/auth.js +++ b/client/coral-framework/actions/auth.js @@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error}); export const checkLogin = () => dispatch => { dispatch(checkLoginRequest()); fetch(`${base}/auth`, getInit('GET')) - .then(handleResp) + .then((res) => { + if (res.status !== 200) { + throw new Error('not logged in'); + } + + return res.json(); + }) .then(user => dispatch(checkLoginSuccess(user))) .catch(error => dispatch(checkLoginFailure(error))); }; diff --git a/client/coral-framework/actions/config.js b/client/coral-framework/actions/config.js deleted file mode 100644 index d8fd886be..000000000 --- a/client/coral-framework/actions/config.js +++ /dev/null @@ -1,35 +0,0 @@ -import {fromJS} from 'immutable'; - -/** - * Action name constants - */ - -export const FETCH_CONFIG_REQUEST = 'FETCH_CONFIG_REQUEST'; -export const FETCH_CONFIG_FAILED = 'FETCH_CONFIG_FAILED'; -export const FETCH_CONFIG_SUCCESS = 'FETCH_CONFIG_SUCCESS'; - -/** - * Action creators - */ - -export function fetchConfig () { - return (dispatch) => { - - dispatch({type: FETCH_CONFIG_REQUEST}); - - return fetch('/api/v1/settings') - .then( - response => { - return response.ok ? response.json() - : Promise.reject(`${response.status} ${response.statusText}`); - } - ) - .then((json) => { - return dispatch({type: FETCH_CONFIG_SUCCESS, config: fromJS(json)}); - }) - .catch((error) => { - dispatch({type: FETCH_CONFIG_FAILED, error}); - }); - - }; -} diff --git a/client/coral-framework/actions/items.js b/client/coral-framework/actions/items.js index f922280c1..4587b174b 100644 --- a/client/coral-framework/actions/items.js +++ b/client/coral-framework/actions/items.js @@ -1,3 +1,5 @@ +import {getInit, base, handleResp} from '../../coral-framework/helpers/response'; +import {fromJS} from 'immutable'; /* Item Actions */ /** @@ -6,28 +8,9 @@ export const ADD_ITEM = 'ADD_ITEM'; export const UPDATE_ITEM = 'UPDATE_ITEM'; +export const UPDATE_SETTINGS = 'UPDATE_SETTINGS'; export const APPEND_ITEM_ARRAY = 'APPEND_ITEM_ARRAY'; -const getInit = (method, body) => { - const headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json' - }; - - const init = {method, headers}; - if (body) { - init.body = JSON.stringify(body); - } - - return init; -}; - -const responseHandler = response => { - if (response.status === 204) { - return; - } - return response.ok ? response.json() : Promise.reject(`${response.status} ${response.statusText}`); -}; /** * Action creators */ @@ -61,6 +44,7 @@ export const addItem = (item, item_type) => { * id - the id of the item to be posted * property - the property to be updated * value - the value that the property should be set to +* item_type - the type of the item being updated (users, comments, etc) * */ export const updateItem = (id, property, value, item_type) => { @@ -73,6 +57,18 @@ export const updateItem = (id, property, value, item_type) => { }; }; +/* +* Appends data to an array in an item in the local store without posting it to the server +* Useful for adding a recently posted reply to a comment, etc. +* +* @params +* id - the id of the item to be posted +* property - the property to be updated (should be an array) +* value - the value that should be added to the array +* add_to_front - boolean that defines whether value is added at the beginning (unshift) or end (push) +* item_type - the type of the item being updated (users, comments, etc) +* +*/ export const appendItemArray = (id, property, value, add_to_front, item_type) => { return { type: APPEND_ITEM_ARRAY, @@ -99,25 +95,25 @@ export const appendItemArray = (id, property, value, add_to_front, item_type) => */ export function getStream (assetUrl) { return (dispatch) => { - return fetch(`/api/v1/stream?asset_url=${encodeURIComponent(assetUrl)}`) - .then(responseHandler) + return fetch(`${base}/stream?asset_url=${encodeURIComponent(assetUrl)}`) + .then(handleResp) .then((json) => { /* Add items to the store */ - const itemTypes = Object.keys(json); - for (let i = 0; i < itemTypes.length; i++ ) { - if (itemTypes[i] === 'actions') { - for (let j = 0; j < json[itemTypes[i]].length; j++ ) { - let action = json[itemTypes[i]][j]; + Object.keys(json).forEach(type => { + if (type === 'actions') { + json[type].forEach(action => { action.id = `${action.action_type}_${action.item_id}`; dispatch(addItem(action, 'actions')); - } + }); + } else if (type === 'settings') { + dispatch({type: UPDATE_SETTINGS, config: fromJS(json[type])}); } else { - for (let j = 0; j < json[itemTypes[i]].length; j++ ) { - dispatch(addItem(json[itemTypes[i]][j], itemTypes[i])); - } + json[type].forEach(item => { + dispatch(addItem(item, type)); + }); } - } + }); const assetId = json.assets[0].id; @@ -140,15 +136,14 @@ export function getStream (assetUrl) { dispatch(updateItem(assetId, 'comments', rels.rootComments, 'assets')); - const childKeys = Object.keys(rels.childComments); - for (let i = 0; i < childKeys.length; i++ ) { - dispatch(updateItem(childKeys[i], 'children', rels.childComments[childKeys[i]].reverse(), 'comments')); - } + Object.keys(rels.childComments).forEach(key => { + dispatch(updateItem(key, 'children', rels.childComments[key].reverse(), 'comments')); + }); /* Hydrate actions on comments */ - for (let i = 0; i < json.actions.length; i++ ) { - dispatch(updateItem(json.actions[i].item_id, json.actions[i].action_type, json.actions[i].id, 'comments')); - } + json.actions.forEach(action => { + dispatch(updateItem(action.item_id, action.action_type, action.id, 'comments')); + }); return (json); }); @@ -171,8 +166,8 @@ export function getStream (assetUrl) { export function getItemsArray (ids) { return (dispatch) => { - return fetch(`/v1/item/${ids}`, getInit('GET')) - .then(responseHandler) + return fetch(`${base}/item/${ids}`, getInit('GET')) + .then(handleResp) .then((json) => { for (let i = 0; i < json.items.length; i++) { dispatch(addItem(json.items[i])); @@ -201,8 +196,8 @@ export function postItem (item, type, id) { if (id) { item.id = id; } - return fetch(`/api/v1/${type}`, getInit('POST', item)) - .then(responseHandler) + return fetch(`${base}/${type}`, getInit('POST', item)) + .then(handleResp) .then((json) => { dispatch(addItem({...item, id:json.id}, type)); return json.id; @@ -232,8 +227,8 @@ export function postAction (item_id, action_type, user_id, item_type) { user_id }; - return fetch(`/api/v1/${item_type}/${item_id}/actions`, getInit('POST', action)) - .then(responseHandler); + return fetch(`${base}/${item_type}/${item_id}/actions`, getInit('POST', action)) + .then(handleResp); }; } @@ -254,7 +249,7 @@ export function postAction (item_id, action_type, user_id, item_type) { export function deleteAction (action_id) { return () => { - return fetch(`/api/v1/actions/${action_id}`, {method: 'DELETE'}) - .then(responseHandler); + return fetch(`${base}/actions/${action_id}`, {method: 'DELETE'}) + .then(handleResp); }; } diff --git a/client/coral-framework/helpers/response.js b/client/coral-framework/helpers/response.js index bccfc5a04..83f51e3ec 100644 --- a/client/coral-framework/helpers/response.js +++ b/client/coral-framework/helpers/response.js @@ -3,10 +3,10 @@ export const base = '/api/v1'; export const getInit = (method, body) => { let init = { method, - headers: new Headers({ + headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' - }), + }, credentials: 'same-origin' }; diff --git a/client/coral-framework/index.js b/client/coral-framework/index.js index 4c6ae741f..837c8956a 100644 --- a/client/coral-framework/index.js +++ b/client/coral-framework/index.js @@ -1,6 +1,5 @@ import Notification from './modules/notification/Notification'; import store from './store'; -import {fetchConfig} from './actions/config'; import * as itemActions from './actions/items'; import I18n from './modules/i18n/i18n'; import * as notificationActions from './actions/notification'; @@ -9,7 +8,6 @@ import * as authActions from './actions/auth'; export { Notification, store, - fetchConfig, itemActions, I18n, notificationActions, diff --git a/client/coral-framework/reducers/config.js b/client/coral-framework/reducers/config.js index cbc131fe6..6521d92a3 100644 --- a/client/coral-framework/reducers/config.js +++ b/client/coral-framework/reducers/config.js @@ -1,7 +1,7 @@ /* @flow */ import {Map} from 'immutable'; -import * as actions from '../actions/config'; +import * as actions from '../actions/items'; const initialState = Map({ features: Map({}) @@ -9,15 +9,10 @@ const initialState = Map({ export default (state = initialState, action) => { switch(action.type) { - case actions.FETCH_CONFIG_REQUEST: - return state.set('loading', true); - case actions.FETCH_CONFIG_FAILED: - return state.set('loading', false); - - // Override config if worked - case actions.FETCH_CONFIG_SUCCESS: - return action.config.set('loading', false); + // Override config if worked + case actions.UPDATE_SETTINGS: + return action.config; default: return state; diff --git a/kue.js b/kue.js new file mode 100644 index 000000000..e2229d424 --- /dev/null +++ b/kue.js @@ -0,0 +1,11 @@ +const kue = require('kue'); +const redis = require('./redis'); + +module.exports = { + queue: kue.createQueue({ + redis: { + createClientFactory: () => redis.createClient() + } + }), + kue +}; diff --git a/models/asset.js b/models/asset.js index 1092cd3e0..b5321cec3 100644 --- a/models/asset.js +++ b/models/asset.js @@ -3,7 +3,6 @@ const uuid = require('uuid'); const Schema = mongoose.Schema; const AssetSchema = new Schema({ - id: { type: String, default: uuid.v4, @@ -19,12 +18,18 @@ const AssetSchema = new Schema({ type: String, default: 'article' }, - headline: String, - summary: String, + scraped: { + type: Date, + default: null + }, + title: String, + description: String, + image: String, section: String, subsection: String, - authors: [String], - publication_date: Date + author: String, + publication_date: Date, + modified_date: Date }, { versionKey: false, timestamps: { @@ -36,80 +41,42 @@ const AssetSchema = new Schema({ /** * Search for assets. Currently only returns all. */ -AssetSchema.statics.search = function(query) { - - return Asset.find(query).exec(); - -}; +AssetSchema.statics.search = (query) => Asset.find(query); /** * Finds an asset by its id. * @param {String} id identifier of the asset (uuid). */ -AssetSchema.statics.findById = function(id) { - - return Asset.findOne({id}).exec(); - -}; +AssetSchema.statics.findById = (id) => Asset.findOne({id}); /** * Finds a asset by its url. * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findByUrl = function(url) { - - return Asset.findOne({'url': url}).exec(); - -}; +AssetSchema.statics.findByUrl = (url) => Asset.findOne({url}); /** * Finds a asset by its url. + * + * NOTE: This function has scalability concerns regarding mongoose's decision + * always write {updated_at: new Date()} on every call to findOneAndUpdate + * even though the update document exactly matches the query document... In + * the future this function should never update, only findOneAndCreate but this + * is not possible with the mongoose driver. + * * @param {String} url identifier of the asset (uuid). */ -AssetSchema.statics.findOrCreateByUrl = function(url) { +AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, { - return Asset.findOne({url}) - .then((asset) => asset ? asset - : Asset.upsert({url})); -}; + // Ensure that if it's new, we return the new object created. + new: true, -/** - * Upserts an asset. -*/ -AssetSchema.statics.upsert = function(data) { - // If an id is not sent, create one. - if (typeof data.id === 'undefined') { - data.id = uuid.v4(); - } + // Perform an upsert in the event that this doesn't exist. + upsert: true, - // Perform the upsert against the id field. - let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec() - .then(() => { - - // Pull the freshly minted asset out and return. - return Asset.findById(data.id); - - }) - .catch((err) => { - - console.error('Error upserting asset.', err); - //return new Promise(); // ??? what do we return on error? - - }); - - return updatePromise; - -}; - -/** - * Remove assets from the db. - * @param {String} query bson query to identify assets to be removed. -*/ -AssetSchema.statics.removeAll = function(query) { - - return Asset.remove(query).exec(); - -}; + // Set the default values if not provided based on the mongoose models. + setDefaultsOnInsert: true +}); const Asset = mongoose.model('Asset', AssetSchema); diff --git a/package.json b/package.json index 9300c347f..bc2f72f20 100644 --- a/package.json +++ b/package.json @@ -4,14 +4,14 @@ "description": "A commenting platform from The Coral Project. https://coralproject.net", "main": "app.js", "scripts": { - "start": "./bin/www", + "start": "./bin/cli serve --jobs", "build": "NODE_ENV=production webpack --config webpack.config.js --bail", "build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch", "lint": "eslint bin/* .", "lint-fix": "eslint . --fix", - "test": "mocha --compilers js:babel-core/register --recursive tests", - "test-watch": "mocha --compilers js:babel-core/register --recursive -w tests", - "embed-start": "NODE_ENV=development npm run build && ./bin/www" + "test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests", + "test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests", + "embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs" }, "config": { "pre-git": { @@ -45,14 +45,16 @@ "express-session": "^1.14.2", "helmet": "^3.1.0", "jsonwebtoken": "^7.1.9", + "kue": "^0.11.5", "lodash": "^4.16.6", + "metascraper": "^1.0.6", "mongoose": "^4.6.5", "morgan": "^1.7.0", + "natural": "^0.4.0", "nodemailer": "^2.6.4", "passport": "^0.3.2", "passport-facebook": "^2.1.1", "passport-local": "^1.0.0", - "natural": "^0.4.0", "prompt": "^1.0.0", "react-linkify": "^0.1.3", "redis": "^2.6.3", diff --git a/redis.js b/redis.js index c37fcc64e..9f67c34bb 100644 --- a/redis.js +++ b/redis.js @@ -2,38 +2,42 @@ const redis = require('redis'); const debug = require('debug')('talk:redis'); const url = process.env.TALK_REDIS_URL || 'redis://localhost'; -const client = redis.createClient(url, { - retry_strategy: function(options) { - if (options.error && options.error.code === 'ECONNREFUSED') { +module.exports = { + createClient() { + let client = redis.createClient(url, { + retry_strategy: function(options) { + if (options.error && options.error.code === 'ECONNREFUSED') { - // End reconnecting on a specific error and flush all commands with a individual error - return new Error('The server refused the connection'); - } - if (options.total_retry_time > 1000 * 60 * 60) { + // End reconnecting on a specific error and flush all commands with a individual error + return new Error('The server refused the connection'); + } + if (options.total_retry_time > 1000 * 60 * 60) { - // End reconnecting after a specific timeout and flush all commands with a individual error - return new Error('Retry time exhausted'); - } + // End reconnecting after a specific timeout and flush all commands with a individual error + return new Error('Retry time exhausted'); + } - if (options.times_connected > 10) { + if (options.times_connected > 10) { - // End reconnecting with built in error - return undefined; - } + // End reconnecting with built in error + return undefined; + } - // reconnect after - return Math.max(options.attempt * 100, 3000); + // reconnect after + return Math.max(options.attempt * 100, 3000); + } + }); + + client.ping((err) => { + if (err) { + console.error('Can\'t ping the redis server!'); + + throw err; + } + + debug('connection established'); + }); + + return client; } -}); - -client.ping((err) => { - if (err) { - console.error('Can\'t ping the redis server!'); - - throw err; - } - - debug('connection established'); -}); - -module.exports = client; +}; diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js index 217c49547..18fd0b7ec 100644 --- a/routes/api/asset/index.js +++ b/routes/api/asset/index.js @@ -1,33 +1,81 @@ const express = require('express'); const router = express.Router(); -const Asset = require('../../../models/asset'); -// Search assets. +const Asset = require('../../../models/asset'); +const scraper = require('../../../services/scraper'); + +// List assets. router.get('/', (req, res, next) => { - let query = {}; + const { + limit = 20, + skip = 0, + sort = 'asc', + field = 'created_at' + } = req.query; - if (typeof req.query.url !== 'undefined') { - query.url = req.query.url; - } + // Find all the assets. + Promise.all([ + Asset + .find({}) + .sort({[field]: (sort === 'asc') ? 1 : -1}) + .skip(skip) + .limit(limit), + Asset.count() + ]) + .then(([result, count]) => { - Asset.search(query) - .then((asset) => { - res.json(asset); - }) - .catch(next); + // Send back the asset data. + res.json({ + result, + count + }); + }) + .catch((err) => { + next(err); + }); }); -// Get an asset by id -router.get('/:id', (req, res, next) => { +// Get an asset by id. +router.get('/:asset_id', (req, res, next) => { - Asset.findById(req.params.id) + // Send back the asset. + Asset + .findById(req.params.asset_id) .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + res.json(asset); }) - .catch(next); + .catch((err) => { + next(err); + }); +}); +// Adds the asset id to the queue to be scraped. +router.post('/:asset_id/scrape', (req, res, next) => { + + // Create a new asset scrape job. + Asset + .findById(req.params.asset_id) + .then((asset) => { + if (!asset) { + return res.status(404).end(); + } + + return scraper.create(asset); + }) + .then((job) => { + + // Send the job back for monitoring. + res.status(201).json(job); + }) + .catch((err) => { + next(err); + }); }); module.exports = router; diff --git a/routes/api/auth/index.js b/routes/api/auth/index.js index b88b6fe12..369e5ec77 100644 --- a/routes/api/auth/index.js +++ b/routes/api/auth/index.js @@ -7,7 +7,18 @@ const router = express.Router(); /** * This returns the user if they are logged in. */ -router.get('/', authorization.needed(), (req, res) => { +router.get('/', (req, res, next) => { + if (req.user) { + return next(); + } + + // When there is no user on the request, then just send back a 204 to this + // request. It's not really "an error" if what they asked for isn't available, + // but it could be. + res.status(204).end(); +}, (req, res) => { + + // Send back the user object. res.json(req.user.toObject()); }); diff --git a/routes/api/index.js b/routes/api/index.js index 8c6014ed8..9b4f0432b 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -3,13 +3,18 @@ const authorization = require('../../middleware/authorization'); const router = express.Router(); -router.use('/asset', require('./asset')); -router.use('/auth', require('./auth')); -router.use('/comments', authorization.needed(), require('./comments')); -router.use('/queue', require('./queue')); +router.use('/asset', authorization.needed('admin'), require('./asset')); router.use('/settings', authorization.needed('admin'), require('./settings')); -router.use('/stream', require('./stream')); -router.use('/user', require('./user')); +router.use('/queue', authorization.needed('admin'), require('./queue')); + +router.use('/comments', authorization.needed(), require('./comments')); router.use('/actions', authorization.needed(), require('./actions')); +router.use('/auth', require('./auth')); +router.use('/stream', require('./stream')); +router.use('/user', require('./user')); + +// Bind the kue handler to the /kue path. +router.use('/kue', authorization.needed('admin'), require('../../kue').kue.app); + module.exports = router; diff --git a/routes/api/queue/index.js b/routes/api/queue/index.js index edc5602ef..f661992f1 100644 --- a/routes/api/queue/index.js +++ b/routes/api/queue/index.js @@ -1,7 +1,6 @@ const express = require('express'); const Comment = require('../../../models/comment'); const Setting = require('../../../models/setting'); -const authorization = require('../../../middleware/authorization'); const router = express.Router(); @@ -13,7 +12,7 @@ const router = express.Router(); // depending on the settings. The :moderation overwrites this settings. // Pre-moderation: New comments are shown in the moderator queues immediately. // Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users. -router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => { +router.get('/comments/pending', (req, res, next) => { Setting.getModerationSetting().then(function({moderation}){ Comment.moderationQueue(moderation).then((comments) => { res.status(200).json(comments); diff --git a/routes/api/settings/index.js b/routes/api/settings/index.js index 6e8c64d4f..910524fb2 100644 --- a/routes/api/settings/index.js +++ b/routes/api/settings/index.js @@ -1,6 +1,5 @@ const express = require('express'); const Setting = require('../../../models/setting'); -const _ = require('lodash'); const router = express.Router(); @@ -8,8 +7,7 @@ router.get('/', (req, res, next) => { Setting .getSettings() .then(settings => { - const whitelist = ['moderation']; - res.json(_.pick(settings, whitelist)); + res.json(settings); }) .catch(next); }); diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index a48548608..7b793cdb6 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -1,55 +1,92 @@ const express = require('express'); const _ = require('lodash'); +const scraper = require('../../../services/scraper'); const Comment = require('../../../models/comment'); const User = require('../../../models/user'); const Action = require('../../../models/action'); const Asset = require('../../../models/asset'); - const Setting = require('../../../models/setting'); const router = express.Router(); -// Find all the comments by a specific asset_url. -// . if pre: get the comments that are accepted. -// . if post: get the comments that are new and accepted. router.get('/', (req, res, next) => { // Get the asset_id for this url (or create it if it doesn't exist) Promise.all([ - Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)), + // Find or create the asset by url. + Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)) + + // Add the found asset to the scraper if it's not already scraped. + .then((asset) => { + if (!asset.scraped) { + return scraper.create(asset).then(() => asset); + } + + return asset; + }), + + // Get the moderation setting from the settings. Setting.getModerationSetting() ]) - .then(([asset, {moderation}]) => { + .then(([asset, settings]) => { // Get the sitewide moderation setting and return the appropriate comments - switch(moderation){ - case 'pre': - return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]); - case 'post': - return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]); - default: - return Promise.reject(new Error('Moderation setting not found.')); + let comments; + if (settings.moderation === 'pre') { + comments = Comment.findAcceptedByAssetId(asset.id); + } else { + comments = Comment.findAcceptedAndNewByAssetId(asset.id); } + + return Promise.all([comments, asset, settings]); }) - // Get all the users and actions for those comments. - .then(([comments, asset]) => { + .then(([comments, asset, settings]) => { + + // Get the user id's from the author id's as a unique array that gets + // sorted. + let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort(); + + // Fetch the users for which there is a comment available for them. + let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : []; + + // Fetch the actions for pretty much everything at this point. + let actions = Action.getActionSummaries(_.uniq([ + + // Actions can be on assets... + asset.id, + + // Comments... + ...comments.map((comment) => comment.id), + + // Or Authors... + ...userIDs + ]), req.user ? req.user.id : false); + return Promise.all([ - [asset], + + // Pass back the asset that we loaded... + asset, + + // It's comments... comments, - User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))), - Action.getActionSummaries(_.uniq([ - asset.id, - ...comments.map((comment) => comment.id), - ...comments.map((comment) => comment.author_id) - ]), req.user ? req.user.id : '') + + // The users who wrote those comments + users, + + // The actions on the above items + actions, + + // And the relevant settings + settings ]); }) - .then(([assets, comments, users, actions]) => { + .then(([asset, comments, users, actions, settings]) => { res.json({ - assets, + assets: [asset], comments, users, - actions + actions, + settings }); }) .catch(error => { diff --git a/routes/api/user/index.js b/routes/api/user/index.js index 3251e0608..43d3b8ac0 100644 --- a/routes/api/user/index.js +++ b/routes/api/user/index.js @@ -16,7 +16,7 @@ router.get('/', authorization.needed('admin'), (req, res, next) => { page = 1, asc = 'false', limit = 50 // Total Per Page - } = req.query; + } = req.query; Promise.all([ User @@ -128,9 +128,10 @@ router.post('/request-password-reset', (req, res, next) => { return mailer.sendSimple(options); }) .then(() => { + // we want to send a 204 regardless of the user being found in the db // if we fail on missing emails, it would reveal if people are registered or not. - res.status(204).send('OK'); + res.status(204).end(); }) .catch(error => { const errorMsg = typeof error === 'string' ? error : error.message; diff --git a/routes/index.js b/routes/index.js index 0c03edcc5..a875d5127 100644 --- a/routes/index.js +++ b/routes/index.js @@ -6,11 +6,11 @@ router.use('/admin', require('./admin')); router.use('/embed', require('./embed')); router.get('/', (req, res) => { - return res.render('article', {title: 'Coral Talk'}); + res.render('article', {title: 'Coral Talk'}); }); router.get('/assets/:asset_title', (req, res) => { - return res.render('article', {title: req.params.asset_title.split('-').join(' ')}); + res.render('article', {title: req.params.asset_title.split('-').join(' ')}); }); module.exports = router; diff --git a/services/scraper.js b/services/scraper.js new file mode 100644 index 000000000..922ef77bc --- /dev/null +++ b/services/scraper.js @@ -0,0 +1,139 @@ +const kue = require('../kue'); +const debug = require('debug')('talk:services:scraper'); +const Asset = require('../models/asset'); +const JOB_NAME = 'scraper'; + +const metascraper = require('metascraper'); + +/** + * Exposes a service object to allow operations to execute against the scraper. + * @type {Object} + */ +const scraper = { + + /** + * creates a new scraper job and scrapes the url when it gets processed. + */ + create(asset) { + return new Promise((resolve, reject) => { + debug(`Creating job for Asset[${asset.id}]`); + + let job = kue.queue + .create(JOB_NAME, { + title: `Scrape for asset ${asset.id}`, + asset_id: asset.id + }) + .attempts(10) + .delay(1000) + .backoff({type: 'exponential'}) + .save((err) => { + if (err) { + return reject(err); + } + + debug(`Created Job[${job.id}] for Asset[${asset.id}]`); + + return resolve(job); + }); + }); + }, + + /** + * Scrapes the given asset for metadata. + */ + scrape(asset) { + return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, { + section: ($) => $('meta[property="article:section"]').attr('content'), + modified: ($) => $('meta[property="article:modified"]').attr('content') + })); + }, + + update(id, meta) { + return Asset.update({id}, { + $set: { + title: meta.title || '', + description: meta.description || '', + image: meta.image ? meta.image : '', + author: meta.author || '', + publication_date: meta.date || '', + modified_date: meta.modified || '', + section: meta.section || '', + scraped: new Date() + } + }); + }, + + /** + * Start the queue processor for the scraper job. + */ + process() { + + debug(`Now processing ${JOB_NAME} jobs`); + + // Process jobs with the processJob function. + kue.queue.process(JOB_NAME, (job, done) => { + + debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + Asset + + // Find the asset, or complain that it doesn't exist. + .findById(job.data.asset_id) + .then((asset) => { + if (!asset) { + throw new Error('asset not found'); + } + + return asset; + }) + + // Scrape the metadata from the asset. + .then(scraper.scrape) + + // Assign the metadata retrieved for the asset to the db. + .then((meta) => { + debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`); + + return scraper.update(job.data.asset_id, meta); + }) + + // Finish the job because we just handled our scraping + updating the + // asset in the database. + .then(() => { + debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`); + done(); + }) + + // Handle errors that occur. + .catch((err) => { + console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err); + + done(err); + }); + }); + }, + + /** + * Shuts down the current queue to ensure that the application can shutdown + * cleanly. + */ + shutdown() { + return new Promise((resolve, reject) => { + + // Shutdown and give the queue 5 seconds to shutdown before we start + // killing jobs. + kue.queue.shutdown(5000, (err) => { + if (err) { + return reject(err); + } + + debug(`Processing for ${JOB_NAME} jobs stopped`); + + resolve(); + }); + }); + } + +}; + +module.exports = scraper; diff --git a/swagger.yaml b/swagger.yaml index fd3b0012b..09adb97d6 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -256,6 +256,86 @@ paths: description: An error occured. schema: $ref: '#/definitions/Error' + /asset: + get: + parameters: + - name: limit + in: query + type: number + format: int32 + description: Limit the listing results + - name: skip + in: query + type: number + format: int32 + description: Skip the listing results + - name: sort + in: query + enum: + - asc + - desc + type: string + description: Sorting method + - name: field + in: query + type: string + description: Field to sort by. + responses: + 200: + description: Assets listed. + schema: + type: object + properties: + count: + type: number + description: Total number of assets found. + result: + type: array + items: + $ref: '#/definitions/Asset' + + /asset/{asset_id}: + get: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 200: + description: The requested asset. + schema: + $ref: '#/definitions/Asset' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + + /asset/{asset_id}/scrape: + post: + parameters: + - name: asset_id + in: path + required: true + type: string + format: uuid + responses: + 201: + description: The job that was created. + schema: + $ref: '#/definitions/Job' + 404: + description: The asset was not found. + 500: + description: An error occured. + schema: + $ref: '#/definitions/Error' + + /stream: get: tags: @@ -420,3 +500,10 @@ definitions: type: object Settings: type: object + Job: + type: object + properties: + id: + format: number + state: + format: string diff --git a/tests/client/coral-framework/store/itemActions.spec.js b/tests/client/coral-framework/store/itemActions.spec.js index 340ec9549..45c41025f 100644 --- a/tests/client/coral-framework/store/itemActions.spec.js +++ b/tests/client/coral-framework/store/itemActions.spec.js @@ -127,6 +127,7 @@ describe('itemActions', () => { 'Accept': 'application/json', 'Content-Type':'application/json' }, + credentials: 'same-origin', body: JSON.stringify(item.data) } ); diff --git a/tests/index.js b/tests/index.js deleted file mode 100644 index 0c3d947ce..000000000 --- a/tests/index.js +++ /dev/null @@ -1,9 +0,0 @@ -const expect = require('chai').expect; - -describe('Comment', () => { - describe('#add', () => { - it('should add a comment', () => { - expect(0).to.be.equal(0); - }); - }); -}); diff --git a/tests/models/action.js b/tests/models/action.js index 87d53b7da..c354b8f83 100644 --- a/tests/models/action.js +++ b/tests/models/action.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Action = require('../../models/action'); const expect = require('chai').expect; diff --git a/tests/models/asset.js b/tests/models/asset.js index d0c3d3429..a5c4ae62b 100644 --- a/tests/models/asset.js +++ b/tests/models/asset.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Asset = require('../../models/asset'); const expect = require('chai').expect; @@ -74,35 +70,4 @@ describe('Asset: model', () => { }); }); }); - - describe('#upsert', ()=> { - it('should insert an asset with no id', () => { - return Asset.upsert({url: 'http://newasset.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id'); - }); - }); - - it('should update an asset when the id exists', () => { - return Asset.upsert({id: 1, url: 'http://new.test.com'}) - .then((asset) => { - expect(asset).to.have.property('id') - .and.to.equal('1'); - expect(asset).to.have.property('url') - .and.to.equal('http://new.test.com'); - }); - }); - }); - - describe('#removeAll', ()=> { - it('should insert an asset with no id', () => { - return Asset.removeAll({id:1}) - .then(() => { - return Asset.findById(1); - }) - .then((result) => { - expect(result).to.be.null; - }); - }); - }); }); diff --git a/tests/models/comment.js b/tests/models/comment.js index f8dd7f9f5..07834a186 100644 --- a/tests/models/comment.js +++ b/tests/models/comment.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const Comment = require('../../models/comment'); const User = require('../../models/user'); const Action = require('../../models/action'); diff --git a/tests/models/setting.js b/tests/models/setting.js index 3f4dc27fa..a36b363fb 100644 --- a/tests/models/setting.js +++ b/tests/models/setting.js @@ -1,7 +1,3 @@ -/* eslint-env node, mocha */ - -require('../utils/mongoose'); - const Setting = require('../../models/setting'); const expect = require('chai').expect; diff --git a/tests/models/user.js b/tests/models/user.js index f77543b4b..c8af92054 100644 --- a/tests/models/user.js +++ b/tests/models/user.js @@ -1,5 +1,3 @@ -require('../utils/mongoose'); - const User = require('../../models/user'); const expect = require('chai').expect; diff --git a/tests/utils/mongoose.js b/tests/mongoose.js similarity index 71% rename from tests/utils/mongoose.js rename to tests/mongoose.js index 1549440a6..9d3ba1195 100644 --- a/tests/utils/mongoose.js +++ b/tests/mongoose.js @@ -1,8 +1,4 @@ -const mongoose = require('../../mongoose'); - -// Ensure the NODE_ENV is set to 'test', -// this is helpful when you would like to change behavior when testing. -process.env.NODE_ENV = 'test'; +const mongoose = require('../mongoose'); beforeEach(function (done) { function clearDB() { diff --git a/tests/utils/passport.js b/tests/passport.js similarity index 90% rename from tests/utils/passport.js rename to tests/passport.js index 401e50b77..2d1c53ab9 100644 --- a/tests/utils/passport.js +++ b/tests/passport.js @@ -1,4 +1,4 @@ -const authorization = require('../../middleware/authorization'); +const authorization = require('../middleware/authorization'); // Add the passport middleware here before it's setup. authorization.middleware.push((req, res, next) => { diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js index 542e88c42..fb67b7b48 100644 --- a/tests/routes/api/assets/index.js +++ b/tests/routes/api/assets/index.js @@ -1,36 +1,10 @@ -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +describe('/assets', () => { -const chai = require('chai'); -const server = require('../../../../app'); + describe('GET', () => { -// Setup chai. -chai.should(); -chai.use(require('chai-http')); + it('should return assets that we search for'); + it('should not return assets that we do not search for'); -describe('Asset: routes', () => { - - describe('/GET Asset', () => { - describe('#get', () => { - it('It should get an empty array when there are no assets.', (done) => { - - chai.request(server) - .get('/api/v1/asset') - .set(passport.inject({roles: ['admin']})) - .end((err, res) => { - - if (err) { - throw new Error(err); - } - - res.should.have.status(200); - res.body.should.be.a('array'); - res.body.length.should.be.eql(0); - done(); - }); - - }); - }); }); }); diff --git a/tests/routes/api/auth/index.js b/tests/routes/api/auth/index.js index dd49a1472..ddca63fe8 100644 --- a/tests/routes/api/auth/index.js +++ b/tests/routes/api/auth/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; diff --git a/tests/routes/api/comments/index.js b/tests/routes/api/comments/index.js index 29775d3d0..ff7f3a9d4 100644 --- a/tests/routes/api/comments/index.js +++ b/tests/routes/api/comments/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/queue/index.js b/tests/routes/api/queue/index.js index 4eff3d384..74137a49a 100644 --- a/tests/routes/api/queue/index.js +++ b/tests/routes/api/queue/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/settings/index.js b/tests/routes/api/settings/index.js index 86feca1eb..9f4466a7f 100644 --- a/tests/routes/api/settings/index.js +++ b/tests/routes/api/settings/index.js @@ -1,7 +1,4 @@ -process.env.NODE_ENV = 'test'; - -require('../../../utils/mongoose'); -const passport = require('../../../utils/passport'); +const passport = require('../../../passport'); const app = require('../../../../app'); const chai = require('chai'); diff --git a/tests/routes/api/stream/index.js b/tests/routes/api/stream/index.js index 009184b6b..2e419a8e1 100644 --- a/tests/routes/api/stream/index.js +++ b/tests/routes/api/stream/index.js @@ -1,5 +1,3 @@ -require('../../../utils/mongoose'); - const app = require('../../../../app'); const chai = require('chai'); const expect = chai.expect; @@ -91,10 +89,11 @@ describe('api/stream: routes', () => { .query({'asset_url': 'http://test.com'}) .then(res => { expect(res).to.have.status(200); - expect(res.body.assets.length).to.equal(1); - expect(res.body.comments.length).to.equal(1); - expect(res.body.users.length).to.equal(1); - expect(res.body.actions.length).to.equal(1); + expect(res.body.assets[0]).to.have.property('url'); + expect(res.body.comments[0]).to.have.property('body'); + expect(res.body.users[0]).to.have.property('displayName'); + expect(res.body.actions[0]).to.have.property('action_type'); + expect(res.body.settings).to.have.property('moderation'); }); }); }); diff --git a/tests/services/scraper.js b/tests/services/scraper.js new file mode 100644 index 000000000..f3b5d006c --- /dev/null +++ b/tests/services/scraper.js @@ -0,0 +1,22 @@ +describe('scraper: services', () => { + describe('#create', () => { + it('should create a new kue job'); + }); + + describe('#scrape', () => { + it('should scrape complete information'); + it('should scrape what it can'); + }); + + describe('#update', () => { + it('should update the database record entries from the meta'); + }); + + describe('#process', () => { + it('should start the processor to scrape assets'); + }); + + describe('#shutdown', () => { + it('should shutdown the job processor'); + }); +}); diff --git a/util.js b/util.js new file mode 100644 index 000000000..6907d90cd --- /dev/null +++ b/util.js @@ -0,0 +1,42 @@ +const util = module.exports = {}; + +/** + * Stores an array of functions that should be executed in the event that the + * application needs to shutdown. + * @type {Array} + */ +util.toshutdown = []; + +/** + * Calls all the shutdown functions and then ends the process. + * @param {Number} [defaultCode=0] default return code upon sucesfull shutdown. + */ +util.shutdown = (defaultCode = 0) => { + Promise + .all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func)) + .then(() => { + process.exit(defaultCode); + }) + .catch((err) => { + console.error(err); + + process.exit(1); + }); +}; + +/** + * Waits until an event is triggered by the node runtime and elevates a series + * of jobs to be ran in the event we need to shutdown. + * @param {Array} jobs Array of promise capable shutdown functions that are + * executed. + */ +util.onshutdown = (jobs) => { + + // Add the new jobs to shutdown to the object reference. + util.toshutdown = util.toshutdown.concat(jobs); +}; + +// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the +// event that we have an external event. +process.on('SIGTERM', () => util.shutdown()); +process.on('SIGINT', () => util.shutdown()); diff --git a/views/article.ejs b/views/article.ejs index 7d168f960..71db59593 100644 --- a/views/article.ejs +++ b/views/article.ejs @@ -1,8 +1,10 @@ - + + +