From 616dd534d9cb1f4a7439f7b83c20780c307afd36 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Mon, 28 Nov 2016 14:05:06 -0700 Subject: [PATCH 1/6] Moved kue redis connector --- app.js | 2 +- cache.js | 8 +++--- kue.js | 11 ++++++++ redis.js | 62 ++++++++++++++++++++++++--------------------- routes/api/index.js | 2 +- services/scraper.js | 9 +++---- 6 files changed, 55 insertions(+), 39 deletions(-) create mode 100644 kue.js diff --git a/app.js b/app.js index 8008cb01e..e84eb4b79 100644 --- a/app.js +++ b/app.js @@ -45,7 +45,7 @@ const session_opts = { }, store: new RedisStore({ ttl: 1800, - client: redis, + client: redis.createClient(), }) }; diff --git a/cache.js b/cache.js index efe689f9c..9345f8cde 100644 --- a/cache.js +++ b/cache.js @@ -1,6 +1,8 @@ const redis = require('./redis'); -const cache = module.exports = {}; +const cache = module.exports = { + client: redis.createClient() +}; /** * This collects a key that may either be an array or a string and creates a @@ -51,7 +53,7 @@ cache.wrap = (key, expiry, work) => { * @return {Promise} */ cache.get = (key) => new Promise((resolve, reject) => { - redis.get(keyfunc(key), (err, reply) => { + cache.client.get(keyfunc(key), (err, reply) => { if (err) { return reject(err); } @@ -87,7 +89,7 @@ cache.set = (key, value, expiry) => new Promise((resolve, reject) => { // Serialize the value as JSON. let reply = JSON.stringify(value); - redis.set(keyfunc(key), reply, 'EX', expiry, (err) => { + cache.client.set(keyfunc(key), reply, 'EX', expiry, (err) => { if (err) { return reject(err); } diff --git a/kue.js b/kue.js new file mode 100644 index 000000000..e2229d424 --- /dev/null +++ b/kue.js @@ -0,0 +1,11 @@ +const kue = require('kue'); +const redis = require('./redis'); + +module.exports = { + queue: kue.createQueue({ + redis: { + createClientFactory: () => redis.createClient() + } + }), + kue +}; diff --git a/redis.js b/redis.js index c37fcc64e..9f67c34bb 100644 --- a/redis.js +++ b/redis.js @@ -2,38 +2,42 @@ const redis = require('redis'); const debug = require('debug')('talk:redis'); const url = process.env.TALK_REDIS_URL || 'redis://localhost'; -const client = redis.createClient(url, { - retry_strategy: function(options) { - if (options.error && options.error.code === 'ECONNREFUSED') { +module.exports = { + createClient() { + let client = redis.createClient(url, { + retry_strategy: function(options) { + if (options.error && options.error.code === 'ECONNREFUSED') { - // End reconnecting on a specific error and flush all commands with a individual error - return new Error('The server refused the connection'); - } - if (options.total_retry_time > 1000 * 60 * 60) { + // End reconnecting on a specific error and flush all commands with a individual error + return new Error('The server refused the connection'); + } + if (options.total_retry_time > 1000 * 60 * 60) { - // End reconnecting after a specific timeout and flush all commands with a individual error - return new Error('Retry time exhausted'); - } + // End reconnecting after a specific timeout and flush all commands with a individual error + return new Error('Retry time exhausted'); + } - if (options.times_connected > 10) { + if (options.times_connected > 10) { - // End reconnecting with built in error - return undefined; - } + // End reconnecting with built in error + return undefined; + } - // reconnect after - return Math.max(options.attempt * 100, 3000); + // reconnect after + return Math.max(options.attempt * 100, 3000); + } + }); + + client.ping((err) => { + if (err) { + console.error('Can\'t ping the redis server!'); + + throw err; + } + + debug('connection established'); + }); + + return client; } -}); - -client.ping((err) => { - if (err) { - console.error('Can\'t ping the redis server!'); - - throw err; - } - - debug('connection established'); -}); - -module.exports = client; +}; diff --git a/routes/api/index.js b/routes/api/index.js index 8da3f791b..9b4f0432b 100644 --- a/routes/api/index.js +++ b/routes/api/index.js @@ -15,6 +15,6 @@ router.use('/stream', require('./stream')); router.use('/user', require('./user')); // Bind the kue handler to the /kue path. -router.use('/kue', authorization.needed('admin'), require('kue').app); +router.use('/kue', authorization.needed('admin'), require('../../kue').kue.app); module.exports = router; diff --git a/services/scraper.js b/services/scraper.js index 665d4fc66..922ef77bc 100644 --- a/services/scraper.js +++ b/services/scraper.js @@ -1,5 +1,4 @@ -const kue = require('kue'); -const queue = kue.createQueue(); +const kue = require('../kue'); const debug = require('debug')('talk:services:scraper'); const Asset = require('../models/asset'); const JOB_NAME = 'scraper'; @@ -19,7 +18,7 @@ const scraper = { return new Promise((resolve, reject) => { debug(`Creating job for Asset[${asset.id}]`); - let job = queue + let job = kue.queue .create(JOB_NAME, { title: `Scrape for asset ${asset.id}`, asset_id: asset.id @@ -72,7 +71,7 @@ const scraper = { debug(`Now processing ${JOB_NAME} jobs`); // Process jobs with the processJob function. - queue.process(JOB_NAME, (job, done) => { + kue.queue.process(JOB_NAME, (job, done) => { debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`); @@ -123,7 +122,7 @@ const scraper = { // Shutdown and give the queue 5 seconds to shutdown before we start // killing jobs. - queue.shutdown(5000, (err) => { + kue.queue.shutdown(5000, (err) => { if (err) { return reject(err); } From ca5049feeb2effd7a8001303b6dd3c67f8cedf6a Mon Sep 17 00:00:00 2001 From: David Jay Date: Mon, 28 Nov 2016 16:52:00 -0500 Subject: [PATCH 2/6] Adding settings to stream. --- routes/api/stream/index.js | 20 +++++++++++--------- tests/routes/api/stream/index.js | 9 +++++---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index acbfe3d77..a22fdd691 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -18,21 +18,21 @@ router.get('/', (req, res, next) => { // Get the asset_id for this url (or create it if it doesn't exist) Promise.all([ Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)), - Setting.getModerationSetting() + Setting.getSettings() ]) - .then(([asset, {moderation}]) => { + .then(([asset, settings]) => { // Get the sitewide moderation setting and return the appropriate comments - switch(moderation){ + switch(settings.moderation){ case 'pre': - return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]); + return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset, settings]); case 'post': - return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]); + return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset, settings]); default: return Promise.reject(new Error('Moderation setting not found.')); } }) // Get all the users and actions for those comments. - .then(([comments, asset]) => { + .then(([comments, asset, settings]) => { return Promise.all([ [asset], comments, @@ -41,15 +41,17 @@ router.get('/', (req, res, next) => { asset.id, ...comments.map((comment) => comment.id), ...comments.map((comment) => comment.author_id) - ])) + ])), + settings ]); }) - .then(([assets, comments, users, actions]) => { + .then(([assets, comments, users, actions, settings]) => { res.json({ assets, comments, users, - actions + actions, + settings }); }) .catch(error => { diff --git a/tests/routes/api/stream/index.js b/tests/routes/api/stream/index.js index 009184b6b..3f032c1d7 100644 --- a/tests/routes/api/stream/index.js +++ b/tests/routes/api/stream/index.js @@ -91,10 +91,11 @@ describe('api/stream: routes', () => { .query({'asset_url': 'http://test.com'}) .then(res => { expect(res).to.have.status(200); - expect(res.body.assets.length).to.equal(1); - expect(res.body.comments.length).to.equal(1); - expect(res.body.users.length).to.equal(1); - expect(res.body.actions.length).to.equal(1); + expect(res.body.assets[0]).to.have.property('url'); + expect(res.body.comments[0]).to.have.property('body'); + expect(res.body.users[0]).to.have.property('displayName'); + expect(res.body.actions[0]).to.have.property('action_type'); + expect(res.body.settings).to.have.property('moderation'); }); }); }); From 823c223cf12c0be75c515915288d06ea8a7dde36 Mon Sep 17 00:00:00 2001 From: David Jay Date: Mon, 28 Nov 2016 17:11:37 -0500 Subject: [PATCH 3/6] Adding settings to redux store from stream. --- client/coral-embed-stream/src/index.js | 4 +-- client/coral-framework/actions/config.js | 35 ----------------------- client/coral-framework/actions/items.js | 17 +++++++++++ client/coral-framework/index.js | 2 -- client/coral-framework/reducers/config.js | 13 +++------ 5 files changed, 22 insertions(+), 49 deletions(-) delete mode 100644 client/coral-framework/actions/config.js diff --git a/client/coral-embed-stream/src/index.js b/client/coral-embed-stream/src/index.js index f2a584ed1..de14edbc7 100644 --- a/client/coral-embed-stream/src/index.js +++ b/client/coral-embed-stream/src/index.js @@ -2,9 +2,7 @@ import React from 'react'; import {render} from 'react-dom'; import CommentStream from './CommentStream'; import {Provider} from 'react-redux'; -import {fetchConfig, store} from '../../coral-framework'; - -store.dispatch(fetchConfig()); +import {store} from '../../coral-framework'; render( diff --git a/client/coral-framework/actions/config.js b/client/coral-framework/actions/config.js deleted file mode 100644 index d8fd886be..000000000 --- a/client/coral-framework/actions/config.js +++ /dev/null @@ -1,35 +0,0 @@ -import {fromJS} from 'immutable'; - -/** - * Action name constants - */ - -export const FETCH_CONFIG_REQUEST = 'FETCH_CONFIG_REQUEST'; -export const FETCH_CONFIG_FAILED = 'FETCH_CONFIG_FAILED'; -export const FETCH_CONFIG_SUCCESS = 'FETCH_CONFIG_SUCCESS'; - -/** - * Action creators - */ - -export function fetchConfig () { - return (dispatch) => { - - dispatch({type: FETCH_CONFIG_REQUEST}); - - return fetch('/api/v1/settings') - .then( - response => { - return response.ok ? response.json() - : Promise.reject(`${response.status} ${response.statusText}`); - } - ) - .then((json) => { - return dispatch({type: FETCH_CONFIG_SUCCESS, config: fromJS(json)}); - }) - .catch((error) => { - dispatch({type: FETCH_CONFIG_FAILED, error}); - }); - - }; -} diff --git a/client/coral-framework/actions/items.js b/client/coral-framework/actions/items.js index f922280c1..5ca358811 100644 --- a/client/coral-framework/actions/items.js +++ b/client/coral-framework/actions/items.js @@ -1,3 +1,4 @@ +import {fromJS} from 'immutable'; /* Item Actions */ /** @@ -6,6 +7,7 @@ export const ADD_ITEM = 'ADD_ITEM'; export const UPDATE_ITEM = 'UPDATE_ITEM'; +export const UPDATE_SETTINGS = 'UPDATE_SETTINGS'; export const APPEND_ITEM_ARRAY = 'APPEND_ITEM_ARRAY'; const getInit = (method, body) => { @@ -61,6 +63,7 @@ export const addItem = (item, item_type) => { * id - the id of the item to be posted * property - the property to be updated * value - the value that the property should be set to +* item_type - the type of the item being updated (users, comments, etc) * */ export const updateItem = (id, property, value, item_type) => { @@ -73,6 +76,18 @@ export const updateItem = (id, property, value, item_type) => { }; }; +/* +* Appends data to an array in an item in the local store without posting it to the server +* Useful for adding a recently posted reply to a comment, etc. +* +* @params +* id - the id of the item to be posted +* property - the property to be updated (should be an array) +* value - the value that should be added to the array +* add_to_front - boolean that defines whether value is added at the beginning (unshift) or end (push) +* item_type - the type of the item being updated (users, comments, etc) +* +*/ export const appendItemArray = (id, property, value, add_to_front, item_type) => { return { type: APPEND_ITEM_ARRAY, @@ -112,6 +127,8 @@ export function getStream (assetUrl) { action.id = `${action.action_type}_${action.item_id}`; dispatch(addItem(action, 'actions')); } + } else if (itemTypes[i] === 'settings') { + return dispatch({type: UPDATE_SETTINGS, config: fromJS(json[itemTypes[i]])}); } else { for (let j = 0; j < json[itemTypes[i]].length; j++ ) { dispatch(addItem(json[itemTypes[i]][j], itemTypes[i])); diff --git a/client/coral-framework/index.js b/client/coral-framework/index.js index 4c6ae741f..837c8956a 100644 --- a/client/coral-framework/index.js +++ b/client/coral-framework/index.js @@ -1,6 +1,5 @@ import Notification from './modules/notification/Notification'; import store from './store'; -import {fetchConfig} from './actions/config'; import * as itemActions from './actions/items'; import I18n from './modules/i18n/i18n'; import * as notificationActions from './actions/notification'; @@ -9,7 +8,6 @@ import * as authActions from './actions/auth'; export { Notification, store, - fetchConfig, itemActions, I18n, notificationActions, diff --git a/client/coral-framework/reducers/config.js b/client/coral-framework/reducers/config.js index cbc131fe6..6521d92a3 100644 --- a/client/coral-framework/reducers/config.js +++ b/client/coral-framework/reducers/config.js @@ -1,7 +1,7 @@ /* @flow */ import {Map} from 'immutable'; -import * as actions from '../actions/config'; +import * as actions from '../actions/items'; const initialState = Map({ features: Map({}) @@ -9,15 +9,10 @@ const initialState = Map({ export default (state = initialState, action) => { switch(action.type) { - case actions.FETCH_CONFIG_REQUEST: - return state.set('loading', true); - case actions.FETCH_CONFIG_FAILED: - return state.set('loading', false); - - // Override config if worked - case actions.FETCH_CONFIG_SUCCESS: - return action.config.set('loading', false); + // Override config if worked + case actions.UPDATE_SETTINGS: + return action.config; default: return state; From 461e80e26045885dd27a69380920dd1f77ed3e3d Mon Sep 17 00:00:00 2001 From: David Jay Date: Mon, 28 Nov 2016 17:43:24 -0500 Subject: [PATCH 4/6] Removing switch in steam endpoint. --- routes/api/stream/index.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/routes/api/stream/index.js b/routes/api/stream/index.js index 17965387b..7b793cdb6 100644 --- a/routes/api/stream/index.js +++ b/routes/api/stream/index.js @@ -31,14 +31,14 @@ router.get('/', (req, res, next) => { ]) .then(([asset, settings]) => { // Get the sitewide moderation setting and return the appropriate comments - switch(settings.moderation){ - case 'pre': - return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset, settings]); - case 'post': - return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset, settings]); - default: - return Promise.reject(new Error('Moderation setting not found.')); + let comments; + if (settings.moderation === 'pre') { + comments = Comment.findAcceptedByAssetId(asset.id); + } else { + comments = Comment.findAcceptedAndNewByAssetId(asset.id); } + + return Promise.all([comments, asset, settings]); }) .then(([comments, asset, settings]) => { From d5ff0f8f281c9a328f1c6acf21756d341eafc122 Mon Sep 17 00:00:00 2001 From: Riley Davis Date: Mon, 28 Nov 2016 16:18:19 -0700 Subject: [PATCH 5/6] keep it DRY. add getInit all over the place. update tests --- client/coral-admin/src/actions/auth.js | 2 +- client/coral-admin/src/actions/community.js | 2 +- client/coral-admin/src/actions/settings.js | 2 +- .../coral-admin/src/services/talk-adapter.js | 2 +- client/coral-framework/actions/items.js | 42 ++++++------------- client/coral-framework/helpers/response.js | 4 +- .../coral-framework/store/itemActions.spec.js | 1 + 7 files changed, 19 insertions(+), 36 deletions(-) diff --git a/client/coral-admin/src/actions/auth.js b/client/coral-admin/src/actions/auth.js index 2c77ffce7..2f8f1041e 100644 --- a/client/coral-admin/src/actions/auth.js +++ b/client/coral-admin/src/actions/auth.js @@ -1,5 +1,5 @@ import * as actions from '../constants/auth'; -import {base, handleResp, getInit} from '../helpers/response'; +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; // Check Login diff --git a/client/coral-admin/src/actions/community.js b/client/coral-admin/src/actions/community.js index 7a4112f8b..5921573d1 100644 --- a/client/coral-admin/src/actions/community.js +++ b/client/coral-admin/src/actions/community.js @@ -9,7 +9,7 @@ import { SET_ROLE } from '../constants/community'; -import {base, getInit, handleResp} from '../helpers/response'; +import {base, getInit, handleResp} from '../../../coral-framework/helpers/response'; export const fetchCommenters = (query = {}) => dispatch => { dispatch(requestFetchCommenters()); diff --git a/client/coral-admin/src/actions/settings.js b/client/coral-admin/src/actions/settings.js index b71a63e39..6a133ddb5 100644 --- a/client/coral-admin/src/actions/settings.js +++ b/client/coral-admin/src/actions/settings.js @@ -1,4 +1,4 @@ -import {base, handleResp, getInit} from '../helpers/response'; +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; export const SETTINGS_LOADING = 'SETTINGS_LOADING'; export const SETTINGS_RECEIVED = 'SETTINGS_RECEIVED'; diff --git a/client/coral-admin/src/services/talk-adapter.js b/client/coral-admin/src/services/talk-adapter.js index eeb799452..6b872d12d 100644 --- a/client/coral-admin/src/services/talk-adapter.js +++ b/client/coral-admin/src/services/talk-adapter.js @@ -1,4 +1,4 @@ -import {base, handleResp, getInit} from '../helpers/response'; +import {base, handleResp, getInit} from '../../../coral-framework/helpers/response'; /** * The adapter is a redux middleware that interecepts the actions that need diff --git a/client/coral-framework/actions/items.js b/client/coral-framework/actions/items.js index 4af6dd1b1..e7d28fbe4 100644 --- a/client/coral-framework/actions/items.js +++ b/client/coral-framework/actions/items.js @@ -1,3 +1,5 @@ +import {getInit, base, handleResp} from '../../coral-framework/helpers/response'; + /* Item Actions */ /** @@ -8,26 +10,6 @@ export const ADD_ITEM = 'ADD_ITEM'; export const UPDATE_ITEM = 'UPDATE_ITEM'; export const APPEND_ITEM_ARRAY = 'APPEND_ITEM_ARRAY'; -const getInit = (method, body) => { - const headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json' - }; - - const init = {method, headers}; - if (body) { - init.body = JSON.stringify(body); - } - - return init; -}; - -const responseHandler = response => { - if (response.status === 204) { - return; - } - return response.ok ? response.json() : Promise.reject(`${response.status} ${response.statusText}`); -}; /** * Action creators */ @@ -99,8 +81,8 @@ export const appendItemArray = (id, property, value, add_to_front, item_type) => */ export function getStream (assetUrl) { return (dispatch) => { - return fetch(`/api/v1/stream?asset_url=${encodeURIComponent(assetUrl)}`) - .then(responseHandler) + return fetch(`${base}/stream?asset_url=${encodeURIComponent(assetUrl)}`) + .then(handleResp) .then((json) => { /* Add items to the store */ @@ -168,8 +150,8 @@ export function getStream (assetUrl) { export function getItemsArray (ids) { return (dispatch) => { - return fetch(`/v1/item/${ids}`, getInit('GET')) - .then(responseHandler) + return fetch(`${base}/item/${ids}`, getInit('GET')) + .then(handleResp) .then((json) => { for (let i = 0; i < json.items.length; i++) { dispatch(addItem(json.items[i])); @@ -198,8 +180,8 @@ export function postItem (item, type, id) { if (id) { item.id = id; } - return fetch(`/api/v1/${type}`, getInit('POST', item)) - .then(responseHandler) + return fetch(`${base}/${type}`, getInit('POST', item)) + .then(handleResp) .then((json) => { dispatch(addItem({...item, id:json.id}, type)); return json.id; @@ -229,8 +211,8 @@ export function postAction (item_id, action_type, user_id, item_type) { user_id }; - return fetch(`/api/v1/${item_type}/${item_id}/actions`, getInit('POST', action)) - .then(responseHandler); + return fetch(`${base}/${item_type}/${item_id}/actions`, getInit('POST', action)) + .then(handleResp); }; } @@ -251,7 +233,7 @@ export function postAction (item_id, action_type, user_id, item_type) { export function deleteAction (action_id) { return () => { - return fetch(`/api/v1/actions/${action_id}`, {method: 'DELETE'}) - .then(responseHandler); + return fetch(`${base}/actions/${action_id}`, {method: 'DELETE'}) + .then(handleResp); }; } diff --git a/client/coral-framework/helpers/response.js b/client/coral-framework/helpers/response.js index bccfc5a04..83f51e3ec 100644 --- a/client/coral-framework/helpers/response.js +++ b/client/coral-framework/helpers/response.js @@ -3,10 +3,10 @@ export const base = '/api/v1'; export const getInit = (method, body) => { let init = { method, - headers: new Headers({ + headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' - }), + }, credentials: 'same-origin' }; diff --git a/tests/client/coral-framework/store/itemActions.spec.js b/tests/client/coral-framework/store/itemActions.spec.js index 340ec9549..45c41025f 100644 --- a/tests/client/coral-framework/store/itemActions.spec.js +++ b/tests/client/coral-framework/store/itemActions.spec.js @@ -127,6 +127,7 @@ describe('itemActions', () => { 'Accept': 'application/json', 'Content-Type':'application/json' }, + credentials: 'same-origin', body: JSON.stringify(item.data) } ); From c0a9ea9905819e94f4b2709a91651e6f5e9c9931 Mon Sep 17 00:00:00 2001 From: David Jay Date: Mon, 28 Nov 2016 18:19:07 -0500 Subject: [PATCH 6/6] Returning all settings from api/v1/settings. --- routes/api/settings/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/routes/api/settings/index.js b/routes/api/settings/index.js index 6e8c64d4f..910524fb2 100644 --- a/routes/api/settings/index.js +++ b/routes/api/settings/index.js @@ -1,6 +1,5 @@ const express = require('express'); const Setting = require('../../../models/setting'); -const _ = require('lodash'); const router = express.Router(); @@ -8,8 +7,7 @@ router.get('/', (req, res, next) => { Setting .getSettings() .then(settings => { - const whitelist = ['moderation']; - res.json(_.pick(settings, whitelist)); + res.json(settings); }) .catch(next); });