wassname/talk
1
0
Fork 0
mirror of https://github.com/wassname/talk.git synced 2026-07-09 23:12:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit of asset queuing (#97)

Browse Source 
* Initial commit of asset queuing

* Addresssing comments
This commit is contained in:
Wyatt Johnson
2016-11-28 11:29:39 -07:00
committed by Dan Zajdband
parent c1a7b95705
commit b0f01cf00f
38 changed files with 832 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.eslintrc.json
+15 -26
View File
@@ -5,25 +5,15 @@
},
"extends": "eslint:recommended",
"rules": {
"indent": [
"error",
"indent": ["error",
2
],
"no-console": [
0
],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"single"
],
"semi": [
"error",
"always"
],
"linebreak-style": ["error", "unix"],
"quotes": ["error", "single"],
"semi": ["error", "always"],
"no-template-curly-in-string": [1],
"no-unsafe-negation": [1],
"array-callback-return": [1],
@@ -35,7 +25,6 @@
"no-throw-literal": [2],
"yoda": [1],
"no-path-concat": [2],
"no-process-exit": [2],
"eol-last": [1],
"no-continue": [1],
"no-nested-ternary": [1],
@@ -46,20 +35,20 @@
"no-const-assign": [2],
"no-duplicate-imports": [2],
"prefer-template": [1],
"comma-spacing": [
"error",
{
"comma-spacing": ["error", {
"after": true
}
],
}],
"no-var": [2],
"no-lonely-if": [2],
"curly": [2],
"no-unused-vars": ["error", { "argsIgnorePattern": "next" }],
"no-multiple-empty-lines": [
"error",
{"max": 1}
],
"newline-per-chained-call": ["error", { "ignoreChainWithDepth": 2 }]
"no-unused-vars": ["error", {
"argsIgnorePattern": "next"
}],
"no-multiple-empty-lines": ["error", {
"max": 1
}],
"newline-per-chained-call": ["error", {
"ignoreChainWithDepth": 2
}]
}
}
bin/cli
+3
View File
@@ -19,7 +19,10 @@ const pkg = require('../package.json');
program
.version(pkg.version)
.command('serve', 'serve the application')
.command('assets', 'interact with assets')
.command('settings', 'work with the application settings')
.command('jobs', 'work with the job queues')
.command('users', 'work with the application auth')
.parse(process.argv);
bin/cli-assets
Executable
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const program = require('commander');
const pkg = require('../package.json');
const Table = require('cli-table');
const Asset = require('../models/asset');
const mongoose = require('../mongoose');
const util = require('../util');
// Register the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
/**
* Lists all the assets registered in the database.
*/
function listAssets() {
Asset
.find({})
.sort({'created_at': 1})
.then((asset) => {
let table = new Table({
head: [
'ID',
'Title',
'URL'
]
});
asset.forEach((asset) => {
table.push([
asset.id,
asset.title ? asset.title : '',
asset.url ? asset.url : ''
]);
});
console.log(table.toString());
util.shutdown();
})
.catch((err) => {
console.error(err);
util.shutdown(1);
});
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.version(pkg.version);
program
.command('list')
.description('list all the assets in the database')
.action(listAssets);
program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/cli-jobs
Executable
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const program = require('commander');
const scraper = require('../services/scraper');
const util = require('../util');
const mongoose = require('../mongoose');
util.onshutdown([
() => mongoose.disconnect()
]);
function processJobs() {
// Start the processor.
scraper.process();
// The scraper only needs to shutdown when the scraper has actually been
// started.
util.onshutdown([
() => scraper.shutdown()
]);
}
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.command('process')
.description('starts job processing')
.action(processJobs);
program.parse(process.argv);
// If there is no command listed, output help.
if (process.argv.length <= 2) {
program.outputHelp();
util.shutdown();
}
bin/cli-serve
Executable
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
const app = require('../app');
const debug = require('debug')('talk:server');
const http = require('http');
const init = require('../init');
const scraper = require('../services/scraper');
const mongoose = require('../mongoose');
const util = require('../util');
/**
* Get port from environment and store in Express.
*/
const port = normalizePort(process.env.TALK_PORT || '3000');
app.set('port', port);
/**
* Create HTTP server.
*/
const server = http.createServer(app);
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? `Pipe ${port}`
: `Port ${port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(`${bind} requires elevated privileges`);
break;
case 'EADDRINUSE':
console.error(`${bind} is already in use`);
break;
}
throw error;
}
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
let addr = server.address();
let bind = typeof addr === 'string'
? `pipe ${ addr}`
: `port ${ addr.port}`;
debug(`Listening on ${ bind}`);
}
/**
* Start the app.
*/
function startApp() {
init().then(() => {
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
});
}
/**
* Module dependencies.
*/
const program = require('commander');
//==============================================================================
// Setting up the program command line arguments.
//==============================================================================
program
.option('-j, --jobs', 'enable job processing on this thread')
.parse(process.argv);
// Start the application serving.
startApp();
// Enable job processing on the thread if enabled.
if (program.jobs) {
// Start the processor.
scraper.process();
}
// Define a safe shutdown function to call in the event we need to shutdown
// because the node hooks are below which will interrupt the shutdown process.
// Shutdown the mongoose connection, the app server, and the scraper.
util.onshutdown([
() => program.jobs ? scraper.shutdown() : null,
() => mongoose.disconnect(),
() => server.close()
]);
bin/cli-settings
+11 -4
View File
@@ -11,6 +11,14 @@ process.env.DEBUG = process.env.TALK_DEBUG;
*/
const program = require('commander');
const mongoose = require('../mongoose');
const Setting = require('../models/setting');
const util = require('../util');
// Regeister the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
//==============================================================================
// Setting up the program command line arguments.
@@ -20,19 +28,17 @@ program
.command('init')
.description('initilizes the talk settings')
.action(() => {
const mongoose = require('../mongoose');
const Setting = require('../models/setting');
const defaults = {id: '1', moderation: 'pre'};
Setting
.update({id: '1'}, {$setOnInsert: defaults}, {upsert: true})
.then(() => {
console.log('Created settings object.');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(`failed to create the settings object ${JSON.stringify(err)}`);
throw new Error(err); // just to be safe
util.shutdown(1);
});
});
@@ -41,4 +47,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/cli-users
+32 -53
View File
@@ -13,14 +13,20 @@ process.env.DEBUG = process.env.TALK_DEBUG;
const program = require('commander');
const pkg = require('../package.json');
const prompt = require('prompt');
const User = require('../models/user');
const mongoose = require('../mongoose');
const util = require('../util');
const Table = require('cli-table');
// Regeister the shutdown criteria.
util.onshutdown([
() => mongoose.disconnect()
]);
/**
* Prompts for input and registers a user based on those.
*/
function createUser(options) {
const User = require('../models/user');
const mongoose = require('../mongoose');
return new Promise((resolve, reject) => {
if (options.flag_mode) {
@@ -74,11 +80,11 @@ function createUser(options) {
})
.then((user) => {
console.log(`Created user ${user.id}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown();
});
}
@@ -86,20 +92,17 @@ function createUser(options) {
* Deletes a user.
*/
function deleteUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.findOneAndRemove({
id: userID
})
.then(() => {
console.log('Deleted user');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown();
});
}
@@ -107,9 +110,6 @@ function deleteUser(userID) {
* Changes the password for a user.
*/
function passwd(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
prompt.start();
prompt.get([
@@ -128,13 +128,13 @@ function passwd(userID) {
], (err, result) => {
if (err) {
console.error(err);
mongoose.disconnect();
util.shutdown();
return;
}
if (result.password !== result.confirmPassword) {
console.error(new Error('Password mismatch'));
mongoose.disconnect();
util.shutdown(1);
return;
}
@@ -142,11 +142,11 @@ function passwd(userID) {
.changePassword(userID, result.password)
.then(() => {
console.log('Password changed.');
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
});
}
@@ -155,9 +155,6 @@ function passwd(userID) {
* Updates the user from the options array.
*/
function updateUser(userID, options) {
const User = require('../models/user');
const mongoose = require('../mongoose');
const updates = [];
if (options.email && typeof options.email === 'string' && options.email.length > 0) {
@@ -189,11 +186,11 @@ function updateUser(userID, options) {
.all(updates.map((q) => q.exec()))
.then(() => {
console.log(`User ${userID} updated.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -201,10 +198,6 @@ function updateUser(userID, options) {
* Lists all the users registered in the database.
*/
function listUsers() {
const Table = require('cli-table');
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.all()
.then((users) => {
@@ -229,11 +222,11 @@ function listUsers() {
});
console.log(table.toString());
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -243,18 +236,15 @@ function listUsers() {
* @param {String} srcUserID id of the user to which is the source of the merge
*/
function mergeUsers(dstUserID, srcUserID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.mergeUsers(dstUserID, srcUserID)
.then(() => {
console.log(`User ${srcUserID} was merged into user ${dstUserID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -264,18 +254,15 @@ function mergeUsers(dstUserID, srcUserID) {
* @param {String} role the role to add
*/
function addRole(userID, role) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.addRoleToUser(userID, role)
.then(() => {
console.log(`Added the ${role} role to User ${userID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -285,18 +272,15 @@ function addRole(userID, role) {
* @param {String} role the role to remove
*/
function removeRole(userID, role) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.removeRoleFromUser(userID, role)
.then(() => {
console.log(`Removed the ${role} role from User ${userID}.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -305,18 +289,15 @@ function removeRole(userID, role) {
* @param {String} userID the ID of a user to disable
*/
function disableUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.disableUser(userID)
.then(() => {
console.log(`User ${userID} was disabled.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -325,18 +306,15 @@ function disableUser(userID) {
* @param {String} userID the ID of a user to enable
*/
function enableUser(userID) {
const User = require('../models/user');
const mongoose = require('../mongoose');
User
.enableUser(userID)
.then(() => {
console.log(`User ${userID} was enabled.`);
mongoose.disconnect();
util.shutdown();
})
.catch((err) => {
console.error(err);
mongoose.disconnect();
util.shutdown(1);
});
}
@@ -408,4 +386,5 @@ program.parse(process.argv);
// If there is no command listed, output help.
if (!process.argv.slice(2).length) {
program.outputHelp();
util.shutdown();
}
bin/www
-97
View File
@@ -1,97 +0,0 @@
#!/usr/bin/env node
/**
* Setup the debug paramater.
*/
process.env.DEBUG = process.env.TALK_DEBUG;
/**
* Module dependencies.
*/
const app = require('../app');
const debug = require('debug')('talk:server');
const http = require('http');
const init = require('../init');
const port = normalizePort(process.env.TALK_PORT || '3000');
let server;
init().then(() => {
/**
* Get port from environment and store in Express.
*/
app.set('port', port);
/**
* Create HTTP server.
*/
server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
server.listen(port);
server.on('error', onError);
server.on('listening', onListening);
});
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
let port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== 'listen') {
throw error;
}
let bind = typeof port === 'string'
? `Pipe ${ port}`
: `Port ${ port}`;
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
console.error(`${bind} requires elevated privileges`);
break;
case 'EADDRINUSE':
console.error(`${bind} is already in use`);
break;
}
throw error;
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
let addr = server.address();
let bind = typeof addr === 'string'
? `pipe ${ addr}`
: `port ${ addr.port}`;
debug(`Listening on ${ bind}`);
}
client/coral-embed-stream/src/CommentStream.js
+6 -2
View File
@@ -61,8 +61,12 @@ class CommentStream extends Component {
// Set up messaging between embedded Iframe an parent component
// Using recommended Pym init code which violates .eslint standards
const pym = new Pym.Child({polling: 100});
const path = /https?\:\/\/([^?]+)/.exec(pym.parentUrl);
this.props.getStream(path && path[1] || window.location);
if (/https?\:\/\/([^?]+)/.test(pym.parentUrl)) {
this.props.getStream(pym.parentUrl);
} else {
this.props.getStream(window.location);
}
}
render () {
client/coral-framework/actions/auth.js
+7 -1
View File
@@ -127,7 +127,13 @@ const checkLoginFailure = error => ({type: actions.CHECK_LOGIN_FAILURE, error});
export const checkLogin = () => dispatch => {
dispatch(checkLoginRequest());
fetch(`${base}/auth`, getInit('GET'))
.then(handleResp)
.then((res) => {
if (res.status !== 200) {
throw new Error('not logged in');
}
return res.json();
})
.then(user => dispatch(checkLoginSuccess(user)))
.catch(error => dispatch(checkLoginFailure(error)));
};
models/asset.js
+28 -61
View File
@@ -3,7 +3,6 @@ const uuid = require('uuid');
const Schema = mongoose.Schema;
const AssetSchema = new Schema({
id: {
type: String,
default: uuid.v4,
@@ -19,12 +18,18 @@ const AssetSchema = new Schema({
type: String,
default: 'article'
},
headline: String,
summary: String,
scraped: {
type: Date,
default: null
},
title: String,
description: String,
image: String,
section: String,
subsection: String,
authors: [String],
publication_date: Date
author: String,
publication_date: Date,
modified_date: Date
}, {
versionKey: false,
timestamps: {
@@ -36,80 +41,42 @@ const AssetSchema = new Schema({
/**
* Search for assets. Currently only returns all.
*/
AssetSchema.statics.search = function(query) {
return Asset.find(query).exec();
};
AssetSchema.statics.search = (query) => Asset.find(query);
/**
* Finds an asset by its id.
* @param {String} id identifier of the asset (uuid).
*/
AssetSchema.statics.findById = function(id) {
return Asset.findOne({id}).exec();
};
AssetSchema.statics.findById = (id) => Asset.findOne({id});
/**
* Finds a asset by its url.
* @param {String} url identifier of the asset (uuid).
*/
AssetSchema.statics.findByUrl = function(url) {
return Asset.findOne({'url': url}).exec();
};
AssetSchema.statics.findByUrl = (url) => Asset.findOne({url});
/**
* Finds a asset by its url.
*
* NOTE: This function has scalability concerns regarding mongoose's decision
* always write {updated_at: new Date()} on every call to findOneAndUpdate
* even though the update document exactly matches the query document... In
* the future this function should never update, only findOneAndCreate but this
* is not possible with the mongoose driver.
*
* @param {String} url identifier of the asset (uuid).
*/
AssetSchema.statics.findOrCreateByUrl = function(url) {
AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, {
return Asset.findOne({url})
.then((asset) => asset ? asset
: Asset.upsert({url}));
};
// Ensure that if it's new, we return the new object created.
new: true,
/**
* Upserts an asset.
*/
AssetSchema.statics.upsert = function(data) {
// If an id is not sent, create one.
if (typeof data.id === 'undefined') {
data.id = uuid.v4();
}
// Perform an upsert in the event that this doesn't exist.
upsert: true,
// Perform the upsert against the id field.
let updatePromise = Asset.update({id: data.id}, data, {upsert: true}).exec()
.then(() => {
// Pull the freshly minted asset out and return.
return Asset.findById(data.id);
})
.catch((err) => {
console.error('Error upserting asset.', err);
//return new Promise(); // ??? what do we return on error?
});
return updatePromise;
};
/**
* Remove assets from the db.
* @param {String} query bson query to identify assets to be removed.
*/
AssetSchema.statics.removeAll = function(query) {
return Asset.remove(query).exec();
};
// Set the default values if not provided based on the mongoose models.
setDefaultsOnInsert: true
});
const Asset = mongoose.model('Asset', AssetSchema);
package.json
+7 -5
View File
@@ -4,14 +4,14 @@
"description": "A commenting platform from The Coral Project. https://coralproject.net",
"main": "app.js",
"scripts": {
"start": "./bin/www",
"start": "./bin/cli serve --jobs",
"build": "NODE_ENV=production webpack --config webpack.config.js --bail",
"build-watch": "NODE_ENV=development webpack --config webpack.config.dev.js --watch",
"lint": "eslint bin/* .",
"lint-fix": "eslint . --fix",
"test": "mocha --compilers js:babel-core/register --recursive tests",
"test-watch": "mocha --compilers js:babel-core/register --recursive -w tests",
"embed-start": "NODE_ENV=development npm run build && ./bin/www"
"test": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive tests",
"test-watch": "NODE_ENV=test mocha --compilers js:babel-core/register --recursive -w tests",
"embed-start": "NODE_ENV=development npm run build && ./bin/cli serve --jobs"
},
"config": {
"pre-git": {
@@ -45,14 +45,16 @@
"express-session": "^1.14.2",
"helmet": "^3.1.0",
"jsonwebtoken": "^7.1.9",
"kue": "^0.11.5",
"lodash": "^4.16.6",
"metascraper": "^1.0.6",
"mongoose": "^4.6.5",
"morgan": "^1.7.0",
"natural": "^0.4.0",
"nodemailer": "^2.6.4",
"passport": "^0.3.2",
"passport-facebook": "^2.1.1",
"passport-local": "^1.0.0",
"natural": "^0.4.0",
"prompt": "^1.0.0",
"react-linkify": "^0.1.3",
"redis": "^2.6.3",
routes/api/asset/index.js
+63 -15
View File
@@ -1,33 +1,81 @@
const express = require('express');
const router = express.Router();
const Asset = require('../../../models/asset');
// Search assets.
const Asset = require('../../../models/asset');
const scraper = require('../../../services/scraper');
// List assets.
router.get('/', (req, res, next) => {
let query = {};
const {
limit = 20,
skip = 0,
sort = 'asc',
field = 'created_at'
} = req.query;
if (typeof req.query.url !== 'undefined') {
query.url = req.query.url;
}
// Find all the assets.
Promise.all([
Asset
.find({})
.sort({[field]: (sort === 'asc') ? 1 : -1})
.skip(skip)
.limit(limit),
Asset.count()
])
.then(([result, count]) => {
Asset.search(query)
.then((asset) => {
res.json(asset);
})
.catch(next);
// Send back the asset data.
res.json({
result,
count
});
})
.catch((err) => {
next(err);
});
});
// Get an asset by id
router.get('/:id', (req, res, next) => {
// Get an asset by id.
router.get('/:asset_id', (req, res, next) => {
Asset.findById(req.params.id)
// Send back the asset.
Asset
.findById(req.params.asset_id)
.then((asset) => {
if (!asset) {
return res.status(404).end();
}
res.json(asset);
})
.catch(next);
.catch((err) => {
next(err);
});
});
// Adds the asset id to the queue to be scraped.
router.post('/:asset_id/scrape', (req, res, next) => {
// Create a new asset scrape job.
Asset
.findById(req.params.asset_id)
.then((asset) => {
if (!asset) {
return res.status(404).end();
}
return scraper.create(asset);
})
.then((job) => {
// Send the job back for monitoring.
res.status(201).json(job);
})
.catch((err) => {
next(err);
});
});
module.exports = router;
routes/api/auth/index.js
+12 -1
View File
@@ -7,7 +7,18 @@ const router = express.Router();
/**
* This returns the user if they are logged in.
*/
router.get('/', authorization.needed(), (req, res) => {
router.get('/', (req, res, next) => {
if (req.user) {
return next();
}
// When there is no user on the request, then just send back a 204 to this
// request. It's not really "an error" if what they asked for isn't available,
// but it could be.
res.status(204).end();
}, (req, res) => {
// Send back the user object.
res.json(req.user.toObject());
});
routes/api/index.js
+11 -6
View File
@@ -3,13 +3,18 @@ const authorization = require('../../middleware/authorization');
const router = express.Router();
router.use('/asset', require('./asset'));
router.use('/auth', require('./auth'));
router.use('/comments', authorization.needed(), require('./comments'));
router.use('/queue', require('./queue'));
router.use('/asset', authorization.needed('admin'), require('./asset'));
router.use('/settings', authorization.needed('admin'), require('./settings'));
router.use('/stream', require('./stream'));
router.use('/user', require('./user'));
router.use('/queue', authorization.needed('admin'), require('./queue'));
router.use('/comments', authorization.needed(), require('./comments'));
router.use('/actions', authorization.needed(), require('./actions'));
router.use('/auth', require('./auth'));
router.use('/stream', require('./stream'));
router.use('/user', require('./user'));
// Bind the kue handler to the /kue path.
router.use('/kue', authorization.needed('admin'), require('kue').app);
module.exports = router;
routes/api/queue/index.js
+1 -2
View File
@@ -1,7 +1,6 @@
const express = require('express');
const Comment = require('../../../models/comment');
const Setting = require('../../../models/setting');
const authorization = require('../../../middleware/authorization');
const router = express.Router();
@@ -13,7 +12,7 @@ const router = express.Router();
// depending on the settings. The :moderation overwrites this settings.
// Pre-moderation: New comments are shown in the moderator queues immediately.
// Post-moderation: New comments do not appear in moderation queues unless they are flagged by other users.
router.get('/comments/pending', authorization.needed('admin'), (req, res, next) => {
router.get('/comments/pending', (req, res, next) => {
Setting.getModerationSetting().then(function({moderation}){
Comment.moderationQueue(moderation).then((comments) => {
res.status(200).json(comments);
routes/api/stream/index.js
+69 -22
View File
@@ -1,52 +1,99 @@
const express = require('express');
const _ = require('lodash');
const scraper = require('../../../services/scraper');
const Comment = require('../../../models/comment');
const User = require('../../../models/user');
const Action = require('../../../models/action');
const Asset = require('../../../models/asset');
const Setting = require('../../../models/setting');
const router = express.Router();
// Find all the comments by a specific asset_url.
// . if pre: get the comments that are accepted.
// . if post: get the comments that are new and accepted.
router.get('/', (req, res, next) => {
// Get the asset_id for this url (or create it if it doesn't exist)
Promise.all([
Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url)),
// Find or create the asset by url.
Asset.findOrCreateByUrl(decodeURIComponent(req.query.asset_url))
// Add the found asset to the scraper if it's not already scraped.
.then((asset) => {
if (!asset.scraped) {
return scraper.create(asset).then(() => asset);
}
return asset;
}),
// Get the moderation setting from the settings.
Setting.getModerationSetting()
])
.then(([asset, {moderation}]) => {
// Get the sitewide moderation setting and return the appropriate comments
switch(moderation){
case 'pre':
return Promise.all([Comment.findAcceptedByAssetId(asset.id), asset]);
case 'post':
return Promise.all([Comment.findAcceptedAndNewByAssetId(asset.id), asset]);
default:
return Promise.reject(new Error('Moderation setting not found.'));
let comments;
if (moderation === 'post') {
comments = Comment.findAcceptedByAssetId(asset.id);
} else {
// Defaults to 'pre' moderation.
comments = Comment.findAcceptedAndNewByAssetId(asset.id);
}
return Promise.all([
// This is the promised component... Fetch the comments based on the
// moderation settings.
comments,
// Send back the reference to the asset.
asset
]);
})
// Get all the users and actions for those comments.
.then(([comments, asset]) => {
// Get the user id's from the author id's as a unique array that gets
// sorted.
let userIDs = _.uniq(comments.map((comment) => comment.author_id)).sort();
// Fetch the users for which there is a comment available for them.
let users = userIDs.length > 0 ? User.findByIdArray(userIDs) : [];
// Fetch the actions for pretty much everything at this point.
let actions = Action.getActionSummaries(_.uniq([
// Actions can be on assets...
asset.id,
// Comments...
...comments.map((comment) => comment.id),
// Or Authors...
...userIDs
]), req.user ? req.user.id : false);
return Promise.all([
[asset],
// Pass back the asset that we loaded...
asset,
// It's comments...
comments,
User.findByIdArray(_.uniq(comments.map((comment) => comment.author_id))),
Action.getActionSummaries(_.uniq([
asset.id,
...comments.map((comment) => comment.id),
...comments.map((comment) => comment.author_id)
]), req.user ? req.user.id : '')
// All the users/authors of those comments...
users,
// And all actions about the asset, comments, and users.
actions
]);
})
.then(([assets, comments, users, actions]) => {
.then(([asset, comments, users, actions]) => {
// Send back the payload containing all this data.
res.json({
assets,
assets: [asset],
comments,
users,
actions
routes/api/user/index.js
+3 -2
View File
@@ -16,7 +16,7 @@ router.get('/', authorization.needed('admin'), (req, res, next) => {
page = 1,
asc = 'false',
limit = 50 // Total Per Page
} = req.query;
} = req.query;
Promise.all([
User
@@ -128,9 +128,10 @@ router.post('/request-password-reset', (req, res, next) => {
return mailer.sendSimple(options);
})
.then(() => {
// we want to send a 204 regardless of the user being found in the db
// if we fail on missing emails, it would reveal if people are registered or not.
res.status(204).send('OK');
res.status(204).end();
})
.catch(error => {
const errorMsg = typeof error === 'string' ? error : error.message;
routes/index.js
+2 -2
View File
@@ -6,11 +6,11 @@ router.use('/admin', require('./admin'));
router.use('/embed', require('./embed'));
router.get('/', (req, res) => {
return res.render('article', {title: 'Coral Talk'});
res.render('article', {title: 'Coral Talk'});
});
router.get('/assets/:asset_title', (req, res) => {
return res.render('article', {title: req.params.asset_title.split('-').join(' ')});
res.render('article', {title: req.params.asset_title.split('-').join(' ')});
});
module.exports = router;
services/scraper.js
+140
View File
@@ -0,0 +1,140 @@
const kue = require('kue');
const queue = kue.createQueue();
const debug = require('debug')('talk:services:scraper');
const Asset = require('../models/asset');
const JOB_NAME = 'scraper';
const metascraper = require('metascraper');
/**
* Exposes a service object to allow operations to execute against the scraper.
* @type {Object}
*/
const scraper = {
/**
* creates a new scraper job and scrapes the url when it gets processed.
*/
create(asset) {
return new Promise((resolve, reject) => {
debug(`Creating job for Asset[${asset.id}]`);
let job = queue
.create(JOB_NAME, {
title: `Scrape for asset ${asset.id}`,
asset_id: asset.id
})
.attempts(10)
.delay(1000)
.backoff({type: 'exponential'})
.save((err) => {
if (err) {
return reject(err);
}
debug(`Created Job[${job.id}] for Asset[${asset.id}]`);
return resolve(job);
});
});
},
/**
* Scrapes the given asset for metadata.
*/
scrape(asset) {
return metascraper.scrapeUrl(asset.url, Object.assign({}, metascraper.RULES, {
section: ($) => $('meta[property="article:section"]').attr('content'),
modified: ($) => $('meta[property="article:modified"]').attr('content')
}));
},
update(id, meta) {
return Asset.update({id}, {
$set: {
title: meta.title || '',
description: meta.description || '',
image: meta.image ? meta.image : '',
author: meta.author || '',
publication_date: meta.date || '',
modified_date: meta.modified || '',
section: meta.section || '',
scraped: new Date()
}
});
},
/**
* Start the queue processor for the scraper job.
*/
process() {
debug(`Now processing ${JOB_NAME} jobs`);
// Process jobs with the processJob function.
queue.process(JOB_NAME, (job, done) => {
debug(`Starting on Job[${job.id}] for Asset[${job.data.asset_id}]`);
Asset
// Find the asset, or complain that it doesn't exist.
.findById(job.data.asset_id)
.then((asset) => {
if (!asset) {
throw new Error('asset not found');
}
return asset;
})
// Scrape the metadata from the asset.
.then(scraper.scrape)
// Assign the metadata retrieved for the asset to the db.
.then((meta) => {
debug(`Scraped ${JSON.stringify(meta)} on Job[${job.id}] for Asset[${job.data.asset_id}]`);
return scraper.update(job.data.asset_id, meta);
})
// Finish the job because we just handled our scraping + updating the
// asset in the database.
.then(() => {
debug(`Finished on Job[${job.id}] for Asset[${job.data.asset_id}]`);
done();
})
// Handle errors that occur.
.catch((err) => {
console.error(`Failed to scrape on Job[${job.id}] for Asset[${job.data.asset_id}]:`, err);
done(err);
});
});
},
/**
* Shuts down the current queue to ensure that the application can shutdown
* cleanly.
*/
shutdown() {
return new Promise((resolve, reject) => {
// Shutdown and give the queue 5 seconds to shutdown before we start
// killing jobs.
queue.shutdown(5000, (err) => {
if (err) {
return reject(err);
}
debug(`Processing for ${JOB_NAME} jobs stopped`);
resolve();
});
});
}
};
module.exports = scraper;
swagger.yaml
+87
View File
@@ -256,6 +256,86 @@ paths:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/asset:
get:
parameters:
- name: limit
in: query
type: number
format: int32
description: Limit the listing results
- name: skip
in: query
type: number
format: int32
description: Skip the listing results
- name: sort
in: query
enum:
- asc
- desc
type: string
description: Sorting method
- name: field
in: query
type: string
description: Field to sort by.
responses:
200:
description: Assets listed.
schema:
type: object
properties:
count:
type: number
description: Total number of assets found.
result:
type: array
items:
$ref: '#/definitions/Asset'
/asset/{asset_id}:
get:
parameters:
- name: asset_id
in: path
required: true
type: string
format: uuid
responses:
200:
description: The requested asset.
schema:
$ref: '#/definitions/Asset'
404:
description: The asset was not found.
500:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/asset/{asset_id}/scrape:
post:
parameters:
- name: asset_id
in: path
required: true
type: string
format: uuid
responses:
201:
description: The job that was created.
schema:
$ref: '#/definitions/Job'
404:
description: The asset was not found.
500:
description: An error occured.
schema:
$ref: '#/definitions/Error'
/stream:
get:
tags:
@@ -420,3 +500,10 @@ definitions:
type: object
Settings:
type: object
Job:
type: object
properties:
id:
format: number
state:
format: string
tests/index.js
-9
View File
@@ -1,9 +0,0 @@
const expect = require('chai').expect;
describe('Comment', () => {
describe('#add', () => {
it('should add a comment', () => {
expect(0).to.be.equal(0);
});
});
});
tests/models/action.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const Action = require('../../models/action');
const expect = require('chai').expect;
tests/models/asset.js
-35
View File
@@ -1,7 +1,3 @@
/* eslint-env node, mocha */
require('../utils/mongoose');
const Asset = require('../../models/asset');
const expect = require('chai').expect;
@@ -74,35 +70,4 @@ describe('Asset: model', () => {
});
});
});
describe('#upsert', ()=> {
it('should insert an asset with no id', () => {
return Asset.upsert({url: 'http://newasset.test.com'})
.then((asset) => {
expect(asset).to.have.property('id');
});
});
it('should update an asset when the id exists', () => {
return Asset.upsert({id: 1, url: 'http://new.test.com'})
.then((asset) => {
expect(asset).to.have.property('id')
.and.to.equal('1');
expect(asset).to.have.property('url')
.and.to.equal('http://new.test.com');
});
});
});
describe('#removeAll', ()=> {
it('should insert an asset with no id', () => {
return Asset.removeAll({id:1})
.then(() => {
return Asset.findById(1);
})
.then((result) => {
expect(result).to.be.null;
});
});
});
});
tests/models/comment.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const Comment = require('../../models/comment');
const User = require('../../models/user');
const Action = require('../../models/action');
tests/models/setting.js
-4
View File
@@ -1,7 +1,3 @@
/* eslint-env node, mocha */
require('../utils/mongoose');
const Setting = require('../../models/setting');
const expect = require('chai').expect;
tests/models/user.js
-2
View File
@@ -1,5 +1,3 @@
require('../utils/mongoose');
const User = require('../../models/user');
const expect = require('chai').expect;
tests/utils/mongoose.js → tests/mongoose.js
+1 -5
View File
@@ -1,8 +1,4 @@
const mongoose = require('../../mongoose');
// Ensure the NODE_ENV is set to 'test',
// this is helpful when you would like to change behavior when testing.
process.env.NODE_ENV = 'test';
const mongoose = require('../mongoose');
beforeEach(function (done) {
function clearDB() {
tests/utils/passport.js → tests/passport.js
+1 -1
View File
@@ -1,4 +1,4 @@
const authorization = require('../../middleware/authorization');
const authorization = require('../middleware/authorization');
// Add the passport middleware here before it's setup.
authorization.middleware.push((req, res, next) => {
tests/routes/api/assets/index.js
+4 -30
View File
@@ -1,36 +1,10 @@
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
describe('/assets', () => {
const chai = require('chai');
const server = require('../../../../app');
describe('GET', () => {
// Setup chai.
chai.should();
chai.use(require('chai-http'));
it('should return assets that we search for');
it('should not return assets that we do not search for');
describe('Asset: routes', () => {
describe('/GET Asset', () => {
describe('#get', () => {
it('It should get an empty array when there are no assets.', (done) => {
chai.request(server)
.get('/api/v1/asset')
.set(passport.inject({roles: ['admin']}))
.end((err, res) => {
if (err) {
throw new Error(err);
}
res.should.have.status(200);
res.body.should.be.a('array');
res.body.length.should.be.eql(0);
done();
});
});
});
});
});
tests/routes/api/auth/index.js
-2
View File
@@ -1,5 +1,3 @@
require('../../../utils/mongoose');
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
tests/routes/api/comments/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/queue/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/settings/index.js
+1 -4
View File
@@ -1,7 +1,4 @@
process.env.NODE_ENV = 'test';
require('../../../utils/mongoose');
const passport = require('../../../utils/passport');
const passport = require('../../../passport');
const app = require('../../../../app');
const chai = require('chai');
tests/routes/api/stream/index.js
+2 -4
View File
@@ -1,5 +1,3 @@
require('../../../utils/mongoose');
const app = require('../../../../app');
const chai = require('chai');
const expect = chai.expect;
@@ -92,8 +90,8 @@ describe('api/stream: routes', () => {
.then(res => {
expect(res).to.have.status(200);
expect(res.body.assets.length).to.equal(1);
expect(res.body.comments.length).to.equal(1);
expect(res.body.users.length).to.equal(1);
expect(res.body.comments.length).to.equal(2);
expect(res.body.users.length).to.equal(2);
expect(res.body.actions.length).to.equal(1);
});
});
tests/services/scraper.js
+22
View File
@@ -0,0 +1,22 @@
describe('scraper: services', () => {
describe('#create', () => {
it('should create a new kue job');
});
describe('#scrape', () => {
it('should scrape complete information');
it('should scrape what it can');
});
describe('#update', () => {
it('should update the database record entries from the meta');
});
describe('#process', () => {
it('should start the processor to scrape assets');
});
describe('#shutdown', () => {
it('should shutdown the job processor');
});
});
util.js
+42
View File
@@ -0,0 +1,42 @@
const util = module.exports = {};
/**
* Stores an array of functions that should be executed in the event that the
* application needs to shutdown.
* @type {Array}
*/
util.toshutdown = [];
/**
* Calls all the shutdown functions and then ends the process.
* @param {Number} [defaultCode=0] default return code upon sucesfull shutdown.
*/
util.shutdown = (defaultCode = 0) => {
Promise
.all(util.toshutdown.map((func) => func ? func() : null).filter((func) => func))
.then(() => {
process.exit(defaultCode);
})
.catch((err) => {
console.error(err);
process.exit(1);
});
};
/**
* Waits until an event is triggered by the node runtime and elevates a series
* of jobs to be ran in the event we need to shutdown.
* @param {Array} jobs Array of promise capable shutdown functions that are
* executed.
*/
util.onshutdown = (jobs) => {
// Add the new jobs to shutdown to the object reference.
util.toshutdown = util.toshutdown.concat(jobs);
};
// Attach to the SIGTERM + SIGINT handles to ensure a clean shutdown in the
// event that we have an external event.
process.on('SIGTERM', () => util.shutdown());
process.on('SIGINT', () => util.shutdown());
views/article.ejs
+3 -1
View File
@@ -1,8 +1,10 @@
<html>
<head>
<meta property="og:title" content="<%= title %>" />
<meta property="og:author" content="A. J. Ournalist" />
<meta property="og:description" content="A description of this article." />
<meta property="article:author" content="A. J. Ournalist" />
<meta property="og:type" content="article">
<meta property="og:image" content="https://coralproject.net/images/splash-md.jpg">
<meta property="article:published" itemprop="datePublished" content="2016-11-16T11:46:06-05:00" />
<meta property="article:modified" itemprop="dateModified" content="2016-11-16T12:09:44-05:00" />
<meta property="article:section" itemprop="articleSection" content="The Section!" />