wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-07-15 11:26:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into guides

Browse Source
This commit is contained in:
Wyatt Johnson
2016-11-28 17:17:28 -07:00
committed by GitHub
parent 979b55180e 00858e467b
commit ad4066407a
62 changed files with 1060 additions and 628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.eslintrc.json
+15 -26
View File
@@ -5,25 +5,15 @@
},
"extends": "eslint:recommended",
"rules": {
"indent": [
"error",
"indent": ["error",
2
],
"no-console": [
0
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"single"
],
"semi": [
"error",
"always"
],
"linebreak-style": ["error", "unix"],
"quotes": ["error", "single"],
"semi": ["error", "always"],
"no-template-curly-in-string": [1],
"no-unsafe-negation": [1],
"array-callback-return": [1],
@@ -35,7 +25,6 @@
"no-throw-literal": [2],
"yoda": [1],
"no-path-concat": [2],
"no-process-exit": [2],
"eol-last": [1],
"no-continue": [1],
"no-nested-ternary": [1],
@@ -46,20 +35,20 @@
"no-const-assign": [2],
"no-duplicate-imports": [2],
"prefer-template": [1],
"comma-spacing": [
"error",
{
"comma-spacing": ["error", {
"after": true
}
],
}],
"no-var": [2],
"no-lonely-if": [2],
"curly": [2],
"no-unused-vars": ["error", { "argsIgnorePattern": "next" }],
"no-multiple-empty-lines": [
"error",
{"max": 1}
],
"newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }]
"no-unused-vars": ["error", {
"argsIgnorePattern": "next"
}],
"no-multiple-empty-lines": ["error", {
"max": 1
}],
"newline-per-chained-call": ["error", {
"ignoreChainWithDepth": 2
}]
}
}
app.js
+1 -1
View File
@@ -45,7 +45,7 @@ const session_opts = {
},
store: new RedisStore({
ttl: 1800,
client: redis,
client: redis.createClient(),
})
};
bin/cli
+3
View File
@@ -19,7 +19,10 @@ const pkg = require('../package.json');
program
.version(pkg.version)
.command('serve', 'serve the application')
.command('assets', 'interact with assets')
.command('settings', 'work with the application settings')
.command('jobs', 'work with the job queues')
.command('users', 'work with the application auth')
.parse(process.argv);
bin/cli-assets
Executable
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const program = require('commander');
const pkg = require('../package.json');
const Table = require('cli-table');
const Asset = require('../models/asset');
const mongoose = require('../mongoose');
const util = require('../util');
// Register the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
/**
* Lists all the assets registered in the database.
*/
function listAssets() {
Asset
.find({})
.sort({'created_at': 1})
.then((asset) => {
let table = new Table({
head: [
'ID',
'Title',
'URL'
]
});
asset.forEach((asset) => {
table.push([
asset.id,
asset.title ? asset.title : '',
asset.url ? asset.url : ''
]);
});
console.log(table.toString());
util.shutdown();
})
.catch((err) => {
console.error(err);
util.shutdown(1);
});
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.version(pkg.version);
program
.command('list')
.description('list all the assets in the database')
.action(listAssets);
program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/cli-jobs
Executable
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const program = require('commander');
const scraper = require('../services/scraper');
const util = require('../util');
const mongoose = require('../mongoose');
util.onshutdown([
() => mongoose.disconnect()
]);
function processJobs() {
// Start the processor.
scraper.process();
// The scraper only needs to shutdown when the scraper has actually been
// started.
util.onshutdown([
() => scraper.shutdown()
]);
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.command('process')
.description('starts job processing')
.action(processJobs);
program.parse(process.argv);
// If there is no command listed, output help.
if (process.argv.length <= 2) {
program.outputHelp();
util.shutdown();
}
bin/cli-serve
Executable
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
const app = require('../app');
const debug = require('debug')('talk:server');
const http = require('http');
const init = require('../init');
const scraper = require('../services/scraper');
const mongoose = require('../mongoose');
const util = require('../util');
/**
* Get port from environment and store in Express.
*/
const port = normalizePort(process.env.TALK_PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
const server = http.createServer(app);
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? `Pipe ${port}`
: `Port ${port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(`${bind} requires elevated privileges`);
break;
case 'EADDRINUSE':
console.error(`${bind} is already in use`);
break;
}
throw error;
}
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
let addr = server.address();
let bind = typeof addr === 'string'
? `pipe ${ addr}`
: `port ${ addr.port}`;
debug(`Listening on ${ bind}`);
}
/**
* Start the app.
*/
function startApp() {
init().then(() => {
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
});
}
/**
* Module dependencies.
*/
const program = require('commander');
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.option('-j, --jobs', 'enable job processing on this thread')
.parse(process.argv);
// Start the application serving.
startApp();
// Enable job processing on the thread if enabled.
if (program.jobs) {
// Start the processor.
scraper.process();
}
// Define a safe shutdown function to call in the event we need to shutdown
// because the node hooks are below which will interrupt the shutdown process.
// Shutdown the mongoose connection, the app server, and the scraper.
util.onshutdown([
() => program.jobs ? scraper.shutdown() : null,
() => mongoose.disconnect(),
() => server.close()
]);
bin/cli-settings
+11 -4
View File
@@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG;
*/
const program = require('commander');
const mongoose = require('../mongoose');
const Setting = require('../models/setting');
const util = require('../util');
// Regeister the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
//==============================================================================
// Setting up the program command line arguments.
@@ -20,19 +28,17 @@ program
.command('init')
.description('initilizes the talk settings')
.action(() => {
const mongoose = require('../mongoose');
const Setting = require('../models/setting');
const defaults = {id: '1', moderation: 'pre'};
Setting
.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true})
.then(() => {
console.log('Created settings object.');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(`failed to create the settings object ${JSON.stringify(err)}`);
throw new Error(err); // just to be safe
util.shutdown(1);
});
});
@@ -41,4 +47,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/cli-users
+32 -53
View File
@@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG;
const program = require('commander');
const pkg = require('../package.json');
const prompt = require('prompt');
const User = require('../models/user');
const mongoose = require('../mongoose');
const util = require('../util');
const Table = require('cli-table');
// Regeister the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
/**
* Prompts for input and registers a user based on those.
*/
function createUser(options) {
const User = require('../models/user');
const mongoose = require('../mongoose');
return new Promise((resolve, reject) => {
if (options.flag_mode) {
@@ -74,11 +80,11 @@ function createUser(options) {
})
.then((user) => {
console.log(`Created user ${user.id}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown();
});
}
@@ -86,20 +92,17 @@ function createUser(options) {
* Deletes a user.
*/
function deleteUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.findOneAndRemove({
id: userID
})
.then(() => {
console.log('Deleted user');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown();
});
}
@@ -107,9 +110,6 @@ function deleteUser(userID) {
* Changes the password for a user.
*/
function passwd(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
prompt.start();
prompt.get([
@@ -128,13 +128,13 @@ function passwd(userID) {
], (err, result) => {
if (err) {
console.error(err);
mongoose.disconnect();
util.shutdown();
return;
}
if (result.password !== result.confirmPassword) {
console.error(new Error('Password mismatch'));
mongoose.disconnect();
util.shutdown(1);
return;
}
@@ -142,11 +142,11 @@ function passwd(userID) {
.changePassword(userID, result.password)
.then(() => {
console.log('Password changed.');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
});
}
@@ -155,9 +155,6 @@ function passwd(userID) {
* Updates the user from the options array.
*/
function updateUser(userID, options) {
const User = require('../models/user');
const mongoose = require('../mongoose');
const updates = [];
if (options.email && typeof options.email === 'string' && options.email.length > 0) {
@@ -189,11 +186,11 @@ function updateUser(userID, options) {
.all(updates.map((q) => q.exec()))
.then(() => {
console.log(`User ${userID} updated.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -201,10 +198,6 @@ function updateUser(userID, options) {
* Lists all the users registered in the database.
*/
function listUsers() {
const Table = require('cli-table');
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.all()
.then((users) => {
@@ -229,11 +222,11 @@ function listUsers() {
});
console.log(table.toString());
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -243,18 +236,15 @@ function listUsers() {
* @param {String} srcUserID id of the user to which is the source of the merge
*/
function mergeUsers(dstUserID, srcUserID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.mergeUsers(dstUserID, srcUserID)
.then(() => {
console.log(`User ${srcUserID} was merged into user ${dstUserID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -264,18 +254,15 @@ function mergeUsers(dstUserID, srcUserID) {
* @param {String} role the role to add
*/
function addRole(userID, role) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.addRoleToUser(userID, role)
.then(() => {
console.log(`Added the ${role} role to User ${userID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -285,18 +272,15 @@ function addRole(userID, role) {
* @param {String} role the role to remove
*/
function removeRole(userID, role) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.removeRoleFromUser(userID, role)
.then(() => {
console.log(`Removed the ${role} role from User ${userID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -305,18 +289,15 @@ function removeRole(userID, role) {
* @param {String} userID the ID of a user to disable
*/
function disableUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.disableUser(userID)
.then(() => {
console.log(`User ${userID} was disabled.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -325,18 +306,15 @@ function disableUser(userID) {
* @param {String} userID the ID of a user to enable
*/
function enableUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.enableUser(userID)
.then(() => {
console.log(`User ${userID} was enabled.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -408,4 +386,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/www
-97
View File
@@ -1,97 +0,0 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const app = require('../app');
const debug = require('debug')('talk:server');
const http = require('http');
const init = require('../init');
const port = normalizePort(process.env.TALK_PORT || '3000');
let server;
init().then(() => {
/**
* Get port from environment and store in Express.
*/
app.set('port', port);
/**
* Create HTTP server.
*/
server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
});
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? `Pipe ${ port}`
: `Port ${ port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(`${bind} requires elevated privileges`);
break;
case 'EADDRINUSE':
console.error(`${bind} is already in use`);
break;
}
throw error;
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
let addr = server.address();
let bind = typeof addr === 'string'
? `pipe ${ addr}`
: `port ${ addr.port}`;
debug(`Listening on ${ bind}`);
}
cache.js
+5 -3
View File
@@ -1,6 +1,8 @@
const redis = require('./redis');
const cache = module.exports = {};
const cache = module.exports = {
client: redis.createClient()
};
/**
* This collects a key that may either be an array or a string and creates a
@@ -51,7 +53,7 @@ cache.wrap = (key, expiry, work) => {
* @return {Promise}
*/
cache.get = (key) => new Promise((resolve, reject) => {
redis.get(keyfunc(key), (err, reply) => {
cache.client.get(keyfunc(key), (err, reply) => {
if (err) {
return reject(err);
}
@@ -87,7 +89,7 @@ cache.set = (key, value, expiry) => new Promise((resolve, reject) => {
// Serialize the value as JSON.
let reply = JSON.stringify(value);
redis.set(keyfunc(key), reply, 'EX', expiry, (err) => {
cache.client.set(keyfunc(key), reply, 'EX', expiry, (err) => {
if (err) {
return reject(err);
}
client/coral-admin/src/actions/auth.js
+15 -1
View File
@@ -1,5 +1,5 @@
import * as actions from '../constants/auth';
import {base, handleResp, getInit} from '../helpers/response';
import {base, handleResp, getInit} from '../../../coral-framework/helpers/response';
// Check Login
@@ -17,3 +17,17 @@ export const checkLogin = () => dispatch => {
})
.catch(error => dispatch(checkLoginFailure(error)));
};
// LogOut Actions
const logOutRequest = () => ({type: actions.LOGOUT_REQUEST});
const logOutSuccess = () => ({type: actions.LOGOUT_SUCCESS});
const logOutFailure = () => ({type: actions.LOGOUT_FAILURE});
export const logout = () => dispatch => {
dispatch(logOutRequest());
fetch(`${base}/auth`, getInit('DELETE'))
.then(handleResp)
.then(() => dispatch(logOutSuccess()))
.catch(error => dispatch(logOutFailure(error)));
};
client/coral-admin/src/actions/community.js
+1 -1
View File
@@ -9,7 +9,7 @@ import {
SET_ROLE
} from '../constants/community';
import {base, getInit, handleResp} from '../helpers/response';
import {base, getInit, handleResp} from '../../../coral-framework/helpers/response';
export const fetchCommenters = (query = {}) => dispatch => {
dispatch(requestFetchCommenters());
client/coral-admin/src/actions/settings.js
+2 -28
View File
@@ -1,3 +1,5 @@
import {base, handleResp, getInit} from '../../../coral-framework/helpers/response';
export const SETTINGS_LOADING = 'SETTINGS_LOADING';
export const SETTINGS_RECEIVED = 'SETTINGS_RECEIVED';
export const SETTINGS_FETCH_ERROR = 'SETTINGS_FETCH_ERROR';
@@ -8,34 +10,6 @@ export const SAVE_SETTINGS_LOADING = 'SAVE_SETTINGS_LOADING';
export const SAVE_SETTINGS_SUCCESS = 'SAVE_SETTINGS_SUCCESS';
export const SAVE_SETTINGS_FAILED = 'SAVE_SETTINGS_FAILED';
const base = '/api/v1';
const getInit = (method, body) => {
const headers = new Headers({
'Content-Type': 'application/json',
'Accept': 'application/json'
});
const init = {method, headers};
if (method.toLowerCase() !== 'get') {
init.body = JSON.stringify(body);
}
return init;
};
const handleResp = res => {
if (res.status === 401) {
throw new Error('Not Authorized to make this request');
} else if (res.status > 399) {
throw new Error('Error! Status ', res.status);
} else if (res.status === 204) {
return res.text();
} else {
return res.json();
}
};
export const fetchSettings = () => dispatch => {
dispatch({type: SETTINGS_LOADING});
fetch(`${base}/settings`, getInit('GET'))
client/coral-admin/src/components/FullLoading.css
+12
View File
@@ -0,0 +1,12 @@
.layout {
max-width: 800px;
margin: 0 auto;
}
.layout h1 {
font-size: 40px;
}
.layout img {
width: 100%;
}
client/coral-admin/src/components/FullLoading.js
+13
View File
@@ -0,0 +1,13 @@
import React from 'react';
import {Layout} from 'react-mdl';
import styles from './FullLoading.css';
import {CoralLogo} from 'coral-ui';
export const FullLoading = () => (
<Layout fixedDrawer>
<div className={styles.layout} >
<h1>Loading</h1>
<CoralLogo />
</div>
</Layout>
);
client/coral-admin/src/components/ui/Header.css
+29 -3
View File
@@ -1,6 +1,5 @@
.header {
background: #505050;
overflow: hidden;
}
.header > div {
@@ -14,8 +13,35 @@
background: #232323;
}
.version {
.rightPanel {
position: absolute;
right: 0;
width: 50px;
width: 170px;
}
.rightPanel ul {
list-style: none;
line-height: 38px;
}
.rightPanel li {
display: inline-block;
float: right;
margin-left: 15px;
}
.rightPanel .settings {
vertical-align: middle;
border-radius: 3px;
border: solid 1px #9e9e9e;
line-height: 10px;
}
.rightPanel .settings > div {
position: relative;
}
.rightPanel .settings:hover {
background: rgba(158, 158, 158, 0.69);
cursor: pointer;
}
client/coral-admin/src/components/ui/Header.js
+22 -7
View File
@@ -1,21 +1,36 @@
import React from 'react';
import {Navigation, Header} from 'react-mdl';
import {Navigation, Header, IconButton, MenuItem, Menu} from 'react-mdl';
import {Link, IndexLink} from 'react-router';
import styles from './Header.css';
import I18n from 'coral-framework/modules/i18n/i18n';
import translations from '../../translations.json';
import {Logo} from './Logo';
export default () => (
export default ({handleLogout}) => (
<Header className={styles.header}>
<Logo />
<Navigation>
<IndexLink className={styles.navLink} to="/admin" activeClassName={styles.active}>{lang.t('configure.moderate')}</IndexLink>
<Link className={styles.navLink} to="/admin/community" activeClassName={styles.active}>{lang.t('configure.community')}</Link>
<Link className={styles.navLink} to="/admin/configure" activeClassName={styles.active}>{lang.t('configure.configure')}</Link>
<IndexLink className={styles.navLink} to="/admin"
activeClassName={styles.active}>{lang.t('configure.moderate')}</IndexLink>
<Link className={styles.navLink} to="/admin/community"
activeClassName={styles.active}>{lang.t('configure.community')}</Link>
<Link className={styles.navLink} to="/admin/configure"
activeClassName={styles.active}>{lang.t('configure.configure')}</Link>
</Navigation>
<div className={styles.version}>
{`v${process.env.VERSION}`}
<div className={styles.rightPanel}>
<ul>
<li className={styles.settings}>
<div>
<IconButton name="settings" id="menu-settings"/>
<Menu target="menu-settings" align="right">
<MenuItem onClick={handleLogout}>Sign Out</MenuItem>
</Menu>
</div>
</li>
<li>
{`v${process.env.VERSION}`}
</li>
</ul>
</div>
</Header>
);
client/coral-admin/src/components/ui/Layout.js
+2 -2
View File
@@ -4,9 +4,9 @@ import Header from './Header';
import Drawer from './Drawer';
import styles from './Layout.css';
export const Layout = ({children}) => (
export const Layout = ({children, ...props}) => (
<LayoutMDL fixedDrawer>
<Header />
<Header {...props}/>
<Drawer />
<div className={styles.layout} >
{children}
client/coral-admin/src/components/ui/Logo.css
+4 -1
View File
@@ -1,7 +1,9 @@
.logo h1 {
color: #272727;
font-size: 20px;
padding: 0 30px;
margin: 0;
line-height: 60px;
padding: 0 20px;
}
.logo span {
@@ -13,6 +15,7 @@
.logo {
background: #E5E5E5;
height: 100%;
}
client/coral-admin/src/containers/LayoutContainer.js
+10 -16
View File
@@ -1,37 +1,31 @@
import React, {Component} from 'react';
import {connect} from 'react-redux';
import {Layout} from '../components/ui/Layout';
import {checkLogin} from '../actions/auth';
import {NotFound} from '../components/NotFound';
import {checkLogin, logout} from '../actions/auth';
import {FullLoading} from '../components/FullLoading';
import {PermissionRequired} from '../components/PermissionRequired';
class LayoutContainer extends Component {
componentWillMount () {
this.props.checkLogin();
const {checkLogin} = this.props;
checkLogin();
}
render () {
const {isAdmin, loggedIn} = this.props.auth;
if (!loggedIn) {
return <NotFound />;
}
if (!isAdmin && loggedIn) {
return <PermissionRequired />;
}
return <Layout {...this.props} />;
const {isAdmin, loggedIn, loadingUser} = this.props.auth;
if (loadingUser) { return <FullLoading />; }
if (!isAdmin) { return <PermissionRequired />; }
if (isAdmin && loggedIn) { return <Layout {...this.props} />; }
return <FullLoading />;
}
}
LayoutContainer.propTypes = {};
const mapStateToProps = state => ({
auth: state.auth.toJS()
});
const mapDispatchToProps = dispatch => ({
checkLogin: () => dispatch(checkLogin()),
handleLogout: () => dispatch(logout())
});
export default connect(
client/coral-admin/src/reducers/auth.js
+7 -1
View File
@@ -9,19 +9,25 @@ const initialState = Map({
export default function auth (state = initialState, action) {
switch (action.type) {
case actions.CHECK_LOGIN_REQUEST:
return state
.set('loadingUser', true);
case actions.CHECK_LOGIN_FAILURE:
return state
.set('loggedIn', false)
.set('loadingUser', false)
.set('user', null);
case actions.CHECK_LOGIN_SUCCESS:
return state
.set('loggedIn', true)
.set('loadingUser', false)
.set('isAdmin', action.isAdmin)
.set('user', action.user);
case actions.LOGOUT_SUCCESS:
return state
.set('loggedIn', false)
.set('user', null);
.set('user', null)
.set('isAdmin', false);
default :
return state;
}
client/coral-admin/src/services/talk-adapter.js
+15 -21
View File
@@ -1,3 +1,4 @@
import {base, handleResp, getInit} from '../../../coral-framework/helpers/response';
/**
* The adapter is a redux middleware that interecepts the actions that need
@@ -7,9 +8,6 @@
* for the coral but also for wordpress comments, disqus and many more.
*/
// Default headers for json payloads.
const jsonHeader = new Headers({'Content-Type': 'application/json'});
// Intercept redux actions and act over the ones we are interested
export default store => next => action => {
@@ -35,11 +33,11 @@ export default store => next => action => {
const fetchModerationQueueComments = store =>
Promise.all([
fetch('/api/v1/queue/comments/pending'),
fetch('/api/v1/comments?status=rejected'),
fetch('/api/v1/comments?action_type=flag')
fetch(`${base}/queue/comments/pending`, getInit('GET')),
fetch(`${base}/comments?status=rejected`, getInit('GET')),
fetch(`${base}/comments?action_type=flag`, getInit('GET'))
])
.then(res => Promise.all(res.map(r => r.json())))
.then(res => Promise.all(res.map(handleResp)))
.then(res => {
res[2] = res[2].map(comment => { comment.flagged = true; return comment; });
return res.reduce((prev, curr) => prev.concat(curr), []);
@@ -51,26 +49,22 @@ Promise.all([
// Update a comment. Now to update a comment we need to send back the whole object
const updateComment = (store, comment) => {
fetch(`/api/v1/comments/${comment.get('id')}/status`, {
method: 'PUT',
headers: jsonHeader,
body: JSON.stringify({status: comment.get('status')})
})
.then(res => res.json())
fetch(`${base}/comments/${comment.get('id')}/status`, getInit('PUT', {status: comment.get('status')}))
.then(handleResp)
.then(res => store.dispatch({type: 'COMMENT_UPDATE_SUCCESS', res}))
.catch(error => store.dispatch({type: 'COMMENT_UPDATE_FAILED', error}));
};
// Create a new comment
const createComment = (store, name, comment) =>
fetch('/api/v1/comments', {
method: 'POST',
body: JSON.stringify({
const createComment = (store, name, comment) => {
const body = {
status: 'Untouched',
body: comment,
name: name,
createdAt: Date.now()
})
}).then(res => res.json())
.then(res => store.dispatch({type: 'COMMENT_CREATE_SUCCESS', comment: res}))
.catch(error => store.dispatch({type: 'COMMENT_CREATE_FAILED', error}));
};
return fetch(`${base}/comments`, getInit('POST', body))
.then(handleResp)
.then(res => store.dispatch({type: 'COMMENT_CREATE_SUCCESS', comment: res}))
.catch(error => store.dispatch({type: 'COMMENT_CREATE_FAILED', error}));
};
client/coral-embed-stream/src/CommentStream.js
+6 -2
View File
@@ -61,8 +61,12 @@ class CommentStream extends Component {
// Set up messaging between embedded Iframe an parent component
// Using recommended Pym init code which violates .eslint standards
const pym = new Pym.Child({polling: 100});
const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl);
this.props.getStream(path && path[1] || window.location);
if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) {
this.props.getStream(pym.parentUrl);
} else {
this.props.getStream(window.location);
}
}
render () {
client/coral-embed-stream/src/index.js
+1 -3
View File
@@ -2,9 +2,7 @@ import React from 'react';
import {render} from 'react-dom';
import CommentStream from './CommentStream';
import {Provider} from 'react-redux';
import {fetchConfig, store} from '../../coral-framework';
store.dispatch(fetchConfig());
import {store} from '../../coral-framework';
render(
<Provider store={store}>
client/coral-framework/actions/auth.js
+7 -1
View File
@@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error});
export const checkLogin = () => dispatch => {
dispatch(checkLoginRequest());
fetch(`${base}/auth`, getInit('GET'))
.then(handleResp)
.then((res) => {
if (res.status !== 200) {
throw new Error('not logged in');
}
return res.json();
})
.then(user => dispatch(checkLoginSuccess(user)))
.catch(error => dispatch(checkLoginFailure(error)));
};
client/coral-framework/actions/config.js
-35
View File
@@ -1,35 +0,0 @@
import {fromJS} from 'immutable';
/**
* Action name constants
*/
export const FETCH_CONFIG_REQUEST = 'FETCH_CONFIG_REQUEST';
export const FETCH_CONFIG_FAILED = 'FETCH_CONFIG_FAILED';
export const FETCH_CONFIG_SUCCESS = 'FETCH_CONFIG_SUCCESS';
/**
* Action creators
*/
export function fetchConfig () {
return (dispatch) => {
dispatch({type: FETCH_CONFIG_REQUEST});
return fetch('/api/v1/settings')
.then(
response => {
return response.ok ? response.json()
: Promise.reject(`${response.status} ${response.statusText}`);
}
)
.then((json) => {
return dispatch({type: FETCH_CONFIG_SUCCESS, config: fromJS(json)});
})
.catch((error) => {
dispatch({type: FETCH_CONFIG_FAILED, error});
});
};
}
client/coral-framework/actions/items.js
+42 -47
View File
@@ -1,3 +1,5 @@
import {getInit, base, handleResp} from '../../coral-framework/helpers/response';
import {fromJS} from 'immutable';
/* Item Actions */
/**
@@ -6,28 +8,9 @@
export const ADD_ITEM = 'ADD_ITEM';
export const UPDATE_ITEM = 'UPDATE_ITEM';
export const UPDATE_SETTINGS = 'UPDATE_SETTINGS';
export const APPEND_ITEM_ARRAY = 'APPEND_ITEM_ARRAY';
const getInit = (method, body) => {
const headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
};
const init = {method, headers};
if (body) {
init.body = JSON.stringify(body);
}
return init;
};
const responseHandler = response => {
if (response.status === 204) {
return;
}
return response.ok ? response.json() : Promise.reject(`${response.status} ${response.statusText}`);
};
/**
* Action creators
*/
@@ -61,6 +44,7 @@ export const addItem = (item, item_type) => {
* id - the id of the item to be posted
* property - the property to be updated
* value - the value that the property should be set to
* item_type - the type of the item being updated (users, comments, etc)
*
*/
export const updateItem = (id, property, value, item_type) => {
@@ -73,6 +57,18 @@ export const updateItem = (id, property, value, item_type) => {
};
};
/*
* Appends data to an array in an item in the local store without posting it to the server
* Useful for adding a recently posted reply to a comment, etc.
*
* @params
* id - the id of the item to be posted
* property - the property to be updated (should be an array)
* value - the value that should be added to the array
* add_to_front - boolean that defines whether value is added at the beginning (unshift) or end (push)
* item_type - the type of the item being updated (users, comments, etc)
*
*/
export const appendItemArray = (id, property, value, add_to_front, item_type) => {
return {
type: APPEND_ITEM_ARRAY,
@@ -99,25 +95,25 @@ export const appendItemArray = (id, property, value, add_to_front, item_type) =>
*/
export function getStream (assetUrl) {
return (dispatch) => {
return fetch(`/api/v1/stream?asset_url=${encodeURIComponent(assetUrl)}`)
.then(responseHandler)
return fetch(`${base}/stream?asset_url=${encodeURIComponent(assetUrl)}`)
.then(handleResp)
.then((json) => {
/* Add items to the store */
const itemTypes = Object.keys(json);
for (let i = 0; i < itemTypes.length; i++ ) {
if (itemTypes[i] === 'actions') {
for (let j = 0; j < json[itemTypes[i]].length; j++ ) {
let action = json[itemTypes[i]][j];
Object.keys(json).forEach(type => {
if (type === 'actions') {
json[type].forEach(action => {
action.id = `${action.action_type}_${action.item_id}`;
dispatch(addItem(action, 'actions'));
}
});
} else if (type === 'settings') {
dispatch({type: UPDATE_SETTINGS, config: fromJS(json[type])});
} else {
for (let j = 0; j < json[itemTypes[i]].length; j++ ) {
dispatch(addItem(json[itemTypes[i]][j], itemTypes[i]));
}
json[type].forEach(item => {
dispatch(addItem(item, type));
});
}
}
});
const assetId = json.assets[0].id;
@@ -140,15 +136,14 @@ export function getStream (assetUrl) {
dispatch(updateItem(assetId, 'comments', rels.rootComments, 'assets'));
const childKeys = Object.keys(rels.childComments);
for (let i = 0; i < childKeys.length; i++ ) {
dispatch(updateItem(childKeys[i], 'children', rels.childComments[childKeys[i]].reverse(), 'comments'));
}
Object.keys(rels.childComments).forEach(key => {
dispatch(updateItem(key, 'children', rels.childComments[key].reverse(), 'comments'));
});
/* Hydrate actions on comments */
for (let i = 0; i < json.actions.length; i++ ) {
dispatch(updateItem(json.actions[i].item_id, json.actions[i].action_type, json.actions[i].id, 'comments'));
}
json.actions.forEach(action => {
dispatch(updateItem(action.item_id, action.action_type, action.id, 'comments'));
});
return (json);
});
@@ -171,8 +166,8 @@ export function getStream (assetUrl) {
export function getItemsArray (ids) {
return (dispatch) => {
return fetch(`/v1/item/${ids}`, getInit('GET'))
.then(responseHandler)
return fetch(`${base}/item/${ids}`, getInit('GET'))
.then(handleResp)
.then((json) => {
for (let i = 0; i < json.items.length; i++) {
dispatch(addItem(json.items[i]));
@@ -201,8 +196,8 @@ export function postItem (item, type, id) {
if (id) {
item.id = id;
}
return fetch(`/api/v1/${type}`, getInit('POST', item))
.then(responseHandler)
return fetch(`${base}/${type}`, getInit('POST', item))
.then(handleResp)
.then((json) => {
dispatch(addItem({...item, id:json.id}, type));
return json.id;
@@ -232,8 +227,8 @@ export function postAction (item_id, action_type, user_id, item_type) {
user_id
};
return fetch(`/api/v1/${item_type}/${item_id}/actions`, getInit('POST', action))
.then(responseHandler);
return fetch(`${base}/${item_type}/${item_id}/actions`, getInit('POST', action))
.then(handleResp);
};
}
@@ -254,7 +249,7 @@ export function postAction (item_id, action_type, user_id, item_type) {
export function deleteAction (action_id) {
return () => {
return fetch(`/api/v1/actions/${action_id}`, {method: 'DELETE'})
.then(responseHandler);
return fetch(`${base}/actions/${action_id}`, {method: 'DELETE'})
.then(handleResp);
};
}
client/coral-framework/helpers/response.js
+2 -2
View File
@@ -3,10 +3,10 @@ export const base = '/api/v1';
export const getInit = (method, body) => {
let init = {
method,
headers: new Headers({
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
}),
},
credentials: 'same-origin'
};
client/coral-framework/index.js
-2
View File
@@ -1,6 +1,5 @@
import Notification from './modules/notification/Notification';
import store from './store';
import {fetchConfig} from './actions/config';
import * as itemActions from './actions/items';
import I18n from './modules/i18n/i18n';
import * as notificationActions from './actions/notification';
@@ -9,7 +8,6 @@ import * as authActions from './actions/auth';
export {
Notification,
store,
fetchConfig,
itemActions,
I18n,
notificationActions,
client/coral-framework/reducers/config.js
+4 -9
View File
@@ -1,7 +1,7 @@
/* @flow */
import {Map} from 'immutable';
import * as actions from '../actions/config';
import * as actions from '../actions/items';
const initialState = Map({
features: Map({})
@@ -9,15 +9,10 @@ const initialState = Map({
export default (state = initialState, action) => {
switch(action.type) {
case actions.FETCH_CONFIG_REQUEST:
return state.set('loading', true);
case actions.FETCH_CONFIG_FAILED:
return state.set('loading', false);
// Override config if worked
case actions.FETCH_CONFIG_SUCCESS:
return action.config.set('loading', false);
// Override config if worked
case actions.UPDATE_SETTINGS:
return action.config;
default:
return state;
kue.js
+11
View File
@@ -0,0 +1,11 @@
const kue = require('kue');
const redis = require('./redis');
module.exports = {
queue: kue.createQueue({
redis: {
createClientFactory: () => redis.createClient()
}
}),
kue
};
models/asset.js
+28 -61
View File
@@ -3,7 +3,6 @@ const uuid = require('uuid');
const Schema = mongoose.Schema;
const AssetSchema = new Schema({
id: {
type: String,
default: uuid.v4,
@@ -19,12 +18,18 @@ const AssetSchema = new Schema({
type: String,
default: 'article'
},
headline: String,
summary: String,
scraped: {
type: Date,
default: null
},
title: String,
description: String,
image: String,
section: String,
subsection: String,
authors: [String],
publication_date: Date
author: String,
publication_date: Date,
modified_date: Date
}, {
versionKey: false,
timestamps: {
@@ -36,80 +41,42 @@ const AssetSchema = new Schema({
/**
* Search for assets. Currently only returns all.
*/
AssetSchema.statics.search = function(query) {
return Asset.find(query).exec();
};
AssetSchema.statics.search = (query) => Asset.find(query);
/**
* Finds an asset by its id.
* @param {String} id identifier of the asset (uuid).
*/
AssetSchema.statics.findById = function(id) {
return Asset.findOne({id}).exec();
};
AssetSchema.statics.findById = (id) => Asset.findOne({id});
/**
* Finds a asset by its url.
* @param {String} url identifier of the asset (uuid).
*/
AssetSchema.statics.findByUrl = function(url) {
return Asset.findOne({'url': url}).exec();
};
AssetSchema.statics.findByUrl = (url) => Asset.findOne({url});
/**
* Finds a asset by its url.
*
* NOTE: This function has scalability concerns regarding mongoose's decision
* always write {updated_at: new Date()} on every call to findOneAndUpdate
* even though the update document exactly matches the query document... In
* the future this function should never update, only findOneAndCreate but this
* is not possible with the mongoose driver.
*
* @param {String} url identifier of the asset (uuid).
*/
AssetSchema.statics.findOrCreateByUrl = function(url) {
AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, {
return Asset.findOne({url})
.then((asset) => asset ? asset
: Asset.upsert({url}));
};
// Ensure that if it's new, we return the new object created.
new: true,
/**
* Upserts an asset.
*/
AssetSchema.statics.upsert = function(data) {
// If an id is not sent, create one.
if (typeof data.id === 'undefined') {
data.id = uuid.v4();
}
// Perform an upsert in the event that this doesn't exist.
upsert: true,
// Perform the upsert against the id field.
let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec()
.then(() => {
// Pull the freshly minted asset out and return.
return Asset.findById(data.id);
})
.catch((err) => {
console.error('Error upserting asset.', err);
//return new Promise(); // ??? what do we return on error?
});
return updatePromise;
};
/**
* Remove assets from the db.
* @param {String} query bson query to identify assets to be removed.
*/
AssetSchema.statics.removeAll = function(query) {
return Asset.remove(query).exec();
};
// Set the default values if not provided based on the mongoose models.
setDefaultsOnInsert: true
});
const Asset = mongoose.model('Asset', AssetSchema);
package.json
+7 -5
View File
@@ -4,14 +4,14 @@
"description": "A commenting platform from The Coral Project. https://coralproject.net",
"main": "app.js",
"scripts": {
"start": "./bin/www",
"start": "./bin/cli serve --jobs",
"build": "NODE_ENV=production webpack --config webpack.config.js --bail",
"build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch",
"lint": "eslint bin/* .",
"lint-fix": "eslint . --fix",
"test": "mocha --compilers js:babel-core/register --recursive tests",
"test-watch": "mocha --compilers js:babel-core/register --recursive -w tests",
"embed-start": "NODE_ENV=development npm run build && ./bin/www"
"test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests",
"test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests",
"embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs"
},
"config": {
"pre-git": {
@@ -45,14 +45,16 @@
"express-session": "^1.14.2",
"helmet": "^3.1.0",
"jsonwebtoken": "^7.1.9",
"kue": "^0.11.5",
"lodash": "^4.16.6",
"metascraper": "^1.0.6",
"mongoose": "^4.6.5",
"morgan": "^1.7.0",
"natural": "^0.4.0",
"nodemailer": "^2.6.4",
"passport": "^0.3.2",
"passport-facebook": "^2.1.1",
"passport-local": "^1.0.0",
"natural": "^0.4.0",
"prompt": "^1.0.0",
"react-linkify": "^0.1.3",
"redis": "^2.6.3",
redis.js
+33 -29
View File
@@ -2,38 +2,42 @@ const redis = require('redis');
const debug = require('debug')('talk:redis');
const url = process.env.TALK_REDIS_URL || 'redis://localhost';
const client = redis.createClient(url, {
retry_strategy: function(options) {
if (options.error && options.error.code === 'ECONNREFUSED') {
module.exports = {
createClient() {
let client = redis.createClient(url, {
retry_strategy: function(options) {
if (options.error && options.error.code === 'ECONNREFUSED') {
// End reconnecting on a specific error and flush all commands with a individual error
return new Error('The server refused the connection');
}
if (options.total_retry_time > 1000 * 60 * 60) {
// End reconnecting on a specific error and flush all commands with a individual error
return new Error('The server refused the connection');
}
if (options.total_retry_time > 1000 * 60 * 60) {
// End reconnecting after a specific timeout and flush all commands with a individual error
return new Error('Retry time exhausted');
}
// End reconnecting after a specific timeout and flush all commands with a individual error
return new Error('Retry time exhausted');
}
if (options.times_connected > 10) {
if (options.times_connected > 10) {
// End reconnecting with built in error
return undefined;
}
// End reconnecting with built in error
return undefined;
}
// reconnect after
return Math.max(options.attempt * 100, 3000);
// reconnect after
return Math.max(options.attempt * 100, 3000);
}
});
client.ping((err) => {
if (err) {
console.error('Can\'t ping the redis server!');
throw err;
}
debug('connection established');
});
return client;
}
});
client.ping((err) => {
if (err) {
console.error('Can\'t ping the redis server!');
throw err;
}
debug('connection established');
});
module.exports = client;
};
routes/api/asset/index.js
+63 -15
View File
@@ -1,33 +1,81 @@
const express = require('express');
const router = express.Router();
const Asset = require('../../../models/asset');
// Search assets.
const Asset = require('../../../models/asset');
const scraper = require('../../../services/scraper');
// List assets.
router.get('/', (req, res, next) => {
let query = {};
const {
limit = 20,
skip = 0,
sort = 'asc',
field = 'created_at'
} = req.query;
if (typeof req.query.url !== 'undefined') {
query.url = req.query.url;
}
// Find all the assets.
Promise.all([
Asset
.find({})
.sort({[field]: (sort === 'asc') ? 1 : -1})
.skip(skip)
.limit(limit),
Asset.count()
])
.then(([result, count]) => {
Asset.search(query)
.then((asset) => {
res.json(asset);
})
.catch(next);
// Send back the asset data.
res.json({
result,
count
});
})
.catch((err) => {
next(err);
});
});
// Get an asset by id
router.get('/:id', (req, res, next) => {
// Get an asset by id.
router.get('/:asset_id', (req, res, next) => {
Asset.findById(req.params.id)
// Send back the asset.
Asset
.findById(req.params.asset_id)
.then((asset) => {
if (!asset) {
return res.status(404).end();
}
res.json(asset);
})
.catch(next);
.catch((err) => {
next(err);
});
});
// Adds the asset id to the queue to be scraped.
router.post('/:asset_id/scrape', (req, res, next) => {
// Create a new asset scrape job.
Asset
.findById(req.params.asset_id)
.then((asset) => {
if (!asset) {
return res.status(404).end();
}
return scraper.create(asset);
})
.then((job) => {
// Send the job back for monitoring.
res.status(201).json(job);
})
.catch((err) => {
next(err);
});
});
module.exports = router;
routes/api/auth/index.js
+12 -1
View File
@@ -7,7 +7,18 @@ const router = express.Router();
/**
* This returns the user if they are logged in.
*/
router.get('/', authorization.needed(), (req, res) => {
router.get('/', (req, res, next) => {
if (req.user) {
return next();
}
// When there is no user on the request, then just send back a 204 to this
// request. It's not really "an error" if what they asked for isn't available,
// but it could be.
res.status(204).end();
}, (req, res) => {
// Send back the user object.
res.json(req.user.toObject());
});
routes/api/index.js
+11 -6
View File
@@ -3,13 +3,18 @@ const authorization = require('../../middleware/authorization');
const router = express.Router();
router.use('/asset', require('./asset'));
router.use('/auth', require('./auth'));
router.use('/comments', authorization.needed(), require('./comments'));
router.use('/queue', require('./queue'));
router.use('/asset', authorization.needed('admin'), require('./asset'));
router.use('/settings', authorization.needed('admin'), require('./settings'));
router.use('/stream', require('./stream'));
router.use('/user', require('./user'));
router.use('/queue', authorization.needed('admin'), require('./queue'));
router.use('/comments', authorization.needed(), require('./comments'));
router.use('/actions', authorization.needed(), require('./actions'));
router.use('/auth', require('./auth'));
router.use('/stream', require('./stream'));
router.use('/user', require('./user'));
// Bind the kue handler to the /kue path.
router.use('/kue', authorization.needed('admin'), require('../../kue').kue.app);
module.exports = router;
routes/api/queue/index.js
+1 -2
View File
@@ -1,7 +1,6 @@
const express = require('express');
const Comment = require('../../../models/comment');
const Setting = require('../../../models/setting');
const authorization = require('../../../middleware/authorization');
const router = express.Router();
@@ -13,7 +12,7 @@ const router = express.Router();
// depending on the settings. The :moderation overwrites this settings.
// Pre-moderation: New comments are shown in the moderator queues immediately.
// Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users.
router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => {
router.get('/comments/pending', (req, res, next) => {
Setting.getModerationSetting().then(function({moderation}){
Comment.moderationQueue(moderation).then((comments) => {
res.status(200).json(comments);
routes/api/settings/index.js
+1 -3
View File
@@ -1,6 +1,5 @@
const express = require('express');
const Setting = require('../../../models/setting');
const _ = require('lodash');
const router = express.Router();
@@ -8,8 +7,7 @@ router.get('/', (req, res, next) => {
Setting
.getSettings()
.then(settings => {
const whitelist = ['moderation'];
res.json(_.pick(settings, whitelist));
res.json(settings);
})
.catch(next);
});
routes/api/stream/index.js
+62 -25
View File
@@ -1,55 +1,92 @@
const express = require('express');
const _ = require('lodash');
const scraper = require('../../../services/scraper');
const Comment = require('../../../models/comment');
const User = require('../../../models/user');
const Action = require('../../../models/action');
const Asset = require('../../../models/asset');
const Setting = require('../../../models/setting');
const router = express.Router();
// Find all the comments by a specific asset_url.
// . if pre: get the comments that are accepted.
// . if post: get the comments that are new and accepted.
router.get('/', (req, res, next) => {
// Get the asset_id for this url (or create it if it doesn't exist)
Promise.all([
Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)),
// Find or create the asset by url.
Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url))
// Add the found asset to the scraper if it's not already scraped.
.then((asset) => {
if (!asset.scraped) {
return scraper.create(asset).then(() => asset);
}
return asset;
}),
// Get the moderation setting from the settings.
Setting.getModerationSetting()
])
.then(([asset, {moderation}]) => {
.then(([asset, settings]) => {
// Get the sitewide moderation setting and return the appropriate comments
switch(moderation){
case 'pre':
return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]);
case 'post':
return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]);
default:
return Promise.reject(new Error('Moderation setting not found.'));
let comments;
if (settings.moderation === 'pre') {
comments = Comment.findAcceptedByAssetId(asset.id);
} else {
comments = Comment.findAcceptedAndNewByAssetId(asset.id);
}
return Promise.all([comments, asset, settings]);
})
// Get all the users and actions for those comments.
.then(([comments, asset]) => {
.then(([comments, asset, settings]) => {
// Get the user id's from the author id's as a unique array that gets
// sorted.
let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort();
// Fetch the users for which there is a comment available for them.
let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : [];
// Fetch the actions for pretty much everything at this point.
let actions = Action.getActionSummaries(_.uniq([
// Actions can be on assets...
asset.id,
// Comments...
...comments.map((comment) => comment.id),
// Or Authors...
...userIDs
]), req.user ? req.user.id : false);
return Promise.all([
[asset],
// Pass back the asset that we loaded...
asset,
// It's comments...
comments,
User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))),
Action.getActionSummaries(_.uniq([
asset.id,
...comments.map((comment) => comment.id),
...comments.map((comment) => comment.author_id)
]), req.user ? req.user.id : '')
// The users who wrote those comments
users,
// The actions on the above items
actions,
// And the relevant settings
settings
]);
})
.then(([assets, comments, users, actions]) => {
.then(([asset, comments, users, actions, settings]) => {
res.json({
assets,
assets: [asset],
comments,
users,
actions
actions,
settings
});
})
.catch(error => {
routes/api/user/index.js
+3 -2
View File
@@ -16,7 +16,7 @@ router.get('/', authorization.needed('admin'), (req, res, next) => {
page = 1,
asc = 'false',
limit = 50 // Total Per Page
} = req.query;
} = req.query;
Promise.all([
User
@@ -128,9 +128,10 @@ router.post('/request-password-reset', (req, res, next) => {
return mailer.sendSimple(options);
})
.then(() => {
// we want to send a 204 regardless of the user being found in the db
// if we fail on missing emails, it would reveal if people are registered or not.
res.status(204).send('OK');
res.status(204).end();
})
.catch(error => {
const errorMsg = typeof error === 'string' ? error : error.message;
routes/index.js
+2 -2
View File
@@ -6,11 +6,11 @@ router.use('/admin', require('./admin'));
router.use('/embed', require('./embed'));
router.get('/', (req, res) => {
return res.render('article', {title: 'Coral Talk'});
res.render('article', {title: 'Coral Talk'});
});
router.get('/assets/:asset_title', (req, res) => {
return res.render('article', {title: req.params.asset_title.split('-').join(' ')});
res.render('article', {title: req.params.asset_title.split('-').join(' ')});
});
module.exports = router;
services/scraper.js
+139
View File
@@ -0,0 +1,139 @@
const kue = require('../kue');
const debug = require('debug')('talk:services:scraper');
const Asset = require('../models/asset');
const JOB_NAME = 'scraper';
const metascraper = require('metascraper');
/**
* Exposes a service object to allow operations to execute against the scraper.
* @type {Object}
*/
const scraper = {
/**
* creates a new scraper job and scrapes the url when it gets processed.
*/
create(asset) {
return new Promise((resolve, reject) => {
debug(`Creating job for Asset[${asset.id}]`);
let job = kue.queue
.create(JOB_NAME, {
title: `Scrape for asset ${asset.id}`,
asset_id: asset.id
})
.attempts(10)
.delay(1000)
.backoff({type: 'exponential'})
.save((err) => {
if (err) {
return reject(err);
}
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
return resolve(job);
});
});
},
/**
* Scrapes the given asset for metadata.
*/
scrape(asset) {
return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, {
section: ($) => $('meta[property="article:section"]').attr('content'),
modified: ($) => $('meta[property="article:modified"]').attr('content')
}));
},
update(id, meta) {
return Asset.update({id}, {
$set: {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
scraped: new Date()
}
});
},
/**
* Start the queue processor for the scraper job.
*/
process() {
debug(`Now processing ${JOB_NAME} jobs`);
// Process jobs with the processJob function.
kue.queue.process(JOB_NAME, (job, done) => {
debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`);
Asset
// Find the asset, or complain that it doesn't exist.
.findById(job.data.asset_id)
.then((asset) => {
if (!asset) {
throw new Error('asset not found');
}
return asset;
})
// Scrape the metadata from the asset.
.then(scraper.scrape)
// Assign the metadata retrieved for the asset to the db.
.then((meta) => {
debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`);
return scraper.update(job.data.asset_id, meta);
})
// Finish the job because we just handled our scraping + updating the
// asset in the database.
.then(() => {
debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`);
done();
})
// Handle errors that occur.
.catch((err) => {
console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err);
done(err);
});
});
},
/**
* Shuts down the current queue to ensure that the application can shutdown
* cleanly.
*/
shutdown() {
return new Promise((resolve, reject) => {
// Shutdown and give the queue 5 seconds to shutdown before we start
// killing jobs.
kue.queue.shutdown(5000, (err) => {
if (err) {
return reject(err);
}
debug(`Processing for ${JOB_NAME} jobs stopped`);
resolve();
});
});
}
};
module.exports = scraper;
swagger.yaml
+87
View File
@@ -256,6 +256,86 @@ paths:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/asset:
get:
parameters:
- name: limit
in: query
type: number
format: int32
description: Limit the listing results
- name: skip
in: query
type: number
format: int32
description: Skip the listing results
- name: sort
in: query
enum:
- asc
- desc
type: string
description: Sorting method
- name: field
in: query
type: string
description: Field to sort by.
responses:
200:
description: Assets listed.
schema:
type: object
properties:
count:
type: number
description: Total number of assets found.
result:
type: array
items:
$ref: '#/definitions/Asset'
/asset/{asset_id}:
get:
parameters:
- name: asset_id
in: path
required: true
type: string
format: uuid
responses:
200:
description: The requested asset.
schema:
$ref: '#/definitions/Asset'
404:
description: The asset was not found.
500:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/asset/{asset_id}/scrape:
post:
parameters:
- name: asset_id
in: path
required: true
type: string
format: uuid
responses:
201:
description: The job that was created.
schema:
$ref: '#/definitions/Job'
404:
description: The asset was not found.
500:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/stream:
get:
tags:
@@ -420,3 +500,10 @@ definitions:
type: object
Settings:
type: object
Job:
type: object
properties:
id:
format: number
state:
format: string
tests/client/coral-framework/store/itemActions.spec.js
+1
View File
@@ -127,6 +127,7 @@ describe('itemActions', () => {
'Accept': 'application/json',
'Content-Type':'application/json'
},
credentials: 'same-origin',
body: JSON.stringify(item.data)
}
);
tests/index.js
-9
View File
@@ -1,9 +0,0 @@
const expect = require('chai').expect;
describe('Comment', () => {
describe('#add', () => {
it('should add a comment', () => {
expect(0).to.be.equal(0);
});
});
});
tests/models/action.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const Action = require('../../models/action');
const expect = require('chai').expect;
tests/models/asset.js
-35
View File
@@ -1,7 +1,3 @@
/* eslint-env node, mocha */
require('../utils/mongoose');
const Asset = require('../../models/asset');
const expect = require('chai').expect;
@@ -74,35 +70,4 @@ describe('Asset: model', () => {
});
});
});
describe('#upsert', ()=> {
it('should insert an asset with no id', () => {
return Asset.upsert({url: 'http://newasset.test.com'})
.then((asset) => {
expect(asset).to.have.property('id');
});
});
it('should update an asset when the id exists', () => {
return Asset.upsert({id: 1, url: 'http://new.test.com'})
.then((asset) => {
expect(asset).to.have.property('id')
.and.to.equal('1');
expect(asset).to.have.property('url')
.and.to.equal('http://new.test.com');
});
});
});
describe('#removeAll', ()=> {
it('should insert an asset with no id', () => {
return Asset.removeAll({id:1})
.then(() => {
return Asset.findById(1);
})
.then((result) => {
expect(result).to.be.null;
});
});
});
});
tests/models/comment.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const Comment = require('../../models/comment');
const User = require('../../models/user');
const Action = require('../../models/action');
tests/models/setting.js
-4
View File
@@ -1,7 +1,3 @@
/* eslint-env node, mocha */
require('../utils/mongoose');
const Setting = require('../../models/setting');
const expect = require('chai').expect;
tests/models/user.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const User = require('../../models/user');
const expect = require('chai').expect;
tests/utils/mongoose.js → tests/mongoose.js
+1 -5
View File
@@ -1,8 +1,4 @@
const mongoose = require('../../mongoose');
// Ensure the NODE_ENV is set to 'test',
// this is helpful when you would like to change behavior when testing.
process.env.NODE_ENV = 'test';
const mongoose = require('../mongoose');
beforeEach(function (done) {
function clearDB() {
tests/utils/passport.js → tests/passport.js
+1 -1
View File
@@ -1,4 +1,4 @@
const authorization = require('../../middleware/authorization');
const authorization = require('../middleware/authorization');
// Add the passport middleware here before it's setup.
authorization.middleware.push((req, res, next) => {
tests/routes/api/assets/index.js
+4 -30
View File
@@ -1,36 +1,10 @@
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
describe('/assets', () => {
const chai = require('chai');
const server = require('../../../../app');
describe('GET', () => {
// Setup chai.
chai.should();
chai.use(require('chai-http'));
it('should return assets that we search for');
it('should not return assets that we do not search for');
describe('Asset: routes', () => {
describe('/GET Asset', () => {
describe('#get', () => {
it('It should get an empty array when there are no assets.', (done) => {
chai.request(server)
.get('/api/v1/asset')
.set(passport.inject({roles: ['admin']}))
.end((err, res) => {
if (err) {
throw new Error(err);
}
res.should.have.status(200);
res.body.should.be.a('array');
res.body.length.should.be.eql(0);
done();
});
});
});
});
});
tests/routes/api/auth/index.js
-2
View File
@@ -1,5 +1,3 @@
require('../../../utils/mongoose');
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
tests/routes/api/comments/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/queue/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/settings/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/stream/index.js
+5 -6
View File
@@ -1,5 +1,3 @@
require('../../../utils/mongoose');
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
@@ -91,10 +89,11 @@ describe('api/stream: routes', () => {
.query({'asset_url': 'http://test.com'})
.then(res => {
expect(res).to.have.status(200);
expect(res.body.assets.length).to.equal(1);
expect(res.body.comments.length).to.equal(1);
expect(res.body.users.length).to.equal(1);
expect(res.body.actions.length).to.equal(1);
expect(res.body.assets[0]).to.have.property('url');
expect(res.body.comments[0]).to.have.property('body');
expect(res.body.users[0]).to.have.property('displayName');
expect(res.body.actions[0]).to.have.property('action_type');
expect(res.body.settings).to.have.property('moderation');
});
});
});
tests/services/scraper.js
+22
View File
@@ -0,0 +1,22 @@
describe('scraper: services', () => {
describe('#create', () => {
it('should create a new kue job');
});
describe('#scrape', () => {
it('should scrape complete information');
it('should scrape what it can');
});
describe('#update', () => {
it('should update the database record entries from the meta');
});
describe('#process', () => {
it('should start the processor to scrape assets');
});
describe('#shutdown', () => {
it('should shutdown the job processor');
});
});
util.js
+42
View File
@@ -0,0 +1,42 @@
const util = module.exports = {};
/**
* Stores an array of functions that should be executed in the event that the
* application needs to shutdown.
* @type {Array}
*/
util.toshutdown = [];
/**
* Calls all the shutdown functions and then ends the process.
* @param {Number} [defaultCode=0] default return code upon sucesfull shutdown.
*/
util.shutdown = (defaultCode = 0) => {
Promise
.all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func))
.then(() => {
process.exit(defaultCode);
})
.catch((err) => {
console.error(err);
process.exit(1);
});
};
/**
* Waits until an event is triggered by the node runtime and elevates a series
* of jobs to be ran in the event we need to shutdown.
* @param {Array} jobs Array of promise capable shutdown functions that are
* executed.
*/
util.onshutdown = (jobs) => {
// Add the new jobs to shutdown to the object reference.
util.toshutdown = util.toshutdown.concat(jobs);
};
// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the
// event that we have an external event.
process.on('SIGTERM', () => util.shutdown());
process.on('SIGINT', () => util.shutdown());
views/article.ejs
+3 -1
View File
@@ -1,8 +1,10 @@
<html>
<head>
<meta property="og:title" content="<%= title %>" />
<meta property="og:author" content="A. J. Ournalist" />
<meta property="og:description" content="A description of this article." />
<meta property="article:author" content="A. J. Ournalist" />
<meta property="og:type" content="article">
<meta property="og:image" content="https://coralproject.net/images/splash-md.jpg">
<meta property="article:published" itemprop="datePublished" content="2016-11-16T11:46:06-05:00" />
<meta property="article:modified" itemprop="dateModified" content="2016-11-16T12:09:44-05:00" />
<meta property="article:section" itemprop="articleSection" content="The Section!" />