diff --git a/bin/cli-assets b/bin/cli-assets index 99965c6d9..9ab36685a 100755 --- a/bin/cli-assets +++ b/bin/cli-assets @@ -12,9 +12,11 @@ process.env.DEBUG = process.env.TALK_DEBUG; const program = require('commander'); const pkg = require('../package.json'); +const parseDuration = require('parse-duration'); const Table = require('cli-table'); const Asset = require('../models/asset'); const mongoose = require('../mongoose'); +const scraper = require('../services/scraper'); const util = require('../util'); // Register the shutdown criteria. @@ -55,6 +57,37 @@ function listAssets() { }); } +function refreshAssets(ageString) { + const now = new Date().getTime(); + const ageMs = parseDuration(ageString); + const age = new Date(now - ageMs); + + Asset.find({ + $or: [ + { + scraped: { + $lte: age + } + }, + { + scraped: null + } + ] + }) + + // Queue all the assets for scraping. + .then((assets) => Promise.all(assets.map(scraper.create))) + + .then(() => { + console.log('Assets were queued to be scraped'); + util.shutdown(); + }) + .catch((err) => { + console.error(err); + util.shutdown(1); + }); +} + //============================================================================== // Setting up the program command line arguments. //============================================================================== @@ -67,6 +100,11 @@ program .description('list all the assets in the database') .action(listAssets); +program + .command('refresh ') + .description('queues the assets that exceed the age requested') + .action(refreshAssets); + program.parse(process.argv); // If there is no command listed, output help. diff --git a/bin/cli-jobs b/bin/cli-jobs index f14276d38..bc320d14b 100755 --- a/bin/cli-jobs +++ b/bin/cli-jobs @@ -14,11 +14,15 @@ const program = require('commander'); const scraper = require('../services/scraper'); const util = require('../util'); const mongoose = require('../mongoose'); +const kue = require('../kue'); util.onshutdown([ () => mongoose.disconnect() ]); +/** + * Starts the job processor. + */ function processJobs() { // Start the processor. @@ -31,6 +35,65 @@ function processJobs() { ]); } +/** + * Removes a single job. + * @param {Object} job the job to be removed + * @return {Promise} + */ +function removeJob(job) { + return new Promise((resolve, reject) => job.remove((err) => { + if (err) { + return reject(err); + } + + return resolve(job); + })); +} + +/** + * Removes the jobs passed in and returns a promise. + * @param {Array} jobs array of jobs + * @return {Promise} + */ +function removeJobs(jobs) { + return Promise.all(jobs.map(removeJob)); +} + +/** + * Get the top n jobs with a specific state. + * @param {String} [state='complete'] state to list jobs by + * @param {Number} limit limit of jobs to load + * @return {Promise} + */ +function rangeJobsByState(state = 'complete', limit) { + return new Promise((resolve, reject) => { + kue.Job.rangeByState(state, 0, limit, 'asc', (err, jobs) => { + if (err) { + return reject(err); + } + + resolve(jobs); + }); + }); +} + +/** + * Cleans up the jobs that are in the queue. + */ +function cleanupJobs(options) { + const n = 100; + + Promise.all([ + rangeJobsByState('complete', n), + options.stuck ? rangeJobsByState('failed', n) : false + ]) + .then((joblists) => joblists.filter((jobs) => jobs).map(removeJobs)) + .then(() => { + util.shutdown(); + console.log('Removed old jobs'); + }); +} + //============================================================================== // Setting up the program command line arguments. //============================================================================== @@ -40,6 +103,12 @@ program .description('starts job processing') .action(processJobs); +program + .command('cleanup') + .option('-s, --stuck', 'cleans up jobs that have been stuck', false) + .description('cleans up inactive jobs') + .action(cleanupJobs); + program.parse(process.argv); // If there is no command listed, output help. diff --git a/models/asset.js b/models/asset.js index b5321cec3..14d5675df 100644 --- a/models/asset.js +++ b/models/asset.js @@ -38,21 +38,32 @@ const AssetSchema = new Schema({ } }); +AssetSchema.index({ + title: 'text', + url: 'text', + description: 'text', + section: 'text', + subsection: 'text', + author: 'text' +}, { + background: true +}); + /** * Search for assets. Currently only returns all. -*/ + */ AssetSchema.statics.search = (query) => Asset.find(query); /** * Finds an asset by its id. * @param {String} id identifier of the asset (uuid). -*/ + */ AssetSchema.statics.findById = (id) => Asset.findOne({id}); /** * Finds a asset by its url. * @param {String} url identifier of the asset (uuid). -*/ + */ AssetSchema.statics.findByUrl = (url) => Asset.findOne({url}); /** @@ -65,7 +76,8 @@ AssetSchema.statics.findByUrl = (url) => Asset.findOne({url}); * is not possible with the mongoose driver. * * @param {String} url identifier of the asset (uuid). -*/ + * @return {Promise} + */ AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, { // Ensure that if it's new, we return the new object created. @@ -78,6 +90,18 @@ AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, { setDefaultsOnInsert: true }); +/** + * Finds assets matching keywords on the model. If `value` is an empty string, + * then it will not even perform a text search query. + * @param {String} value string to search by. + * @return {Promise} + */ +AssetSchema.statics.search = (value) => value.length === 0 ? Asset.find({}) : Asset.find({ + $text: { + $search: value + } +}); + const Asset = mongoose.model('Asset', AssetSchema); module.exports = Asset; diff --git a/package.json b/package.json index bc2f72f20..8d0256af2 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,13 @@ "config": { "pre-git": { "commit-msg": [], - "pre-commit": ["npm run lint", "npm test"], - "pre-push": ["npm test"], + "pre-commit": [ + "npm run lint", + "npm test" + ], + "pre-push": [ + "npm test" + ], "post-commit": [], "post-merge": [] } @@ -26,7 +31,12 @@ "type": "git", "url": "git+https://github.com/coralproject/talk.git" }, - "keywords": ["talk", "coral", "coralproject", "ask"], + "keywords": [ + "talk", + "coral", + "coralproject", + "ask" + ], "author": "", "license": "Apache-2.0", "bugs": { @@ -52,6 +62,7 @@ "morgan": "^1.7.0", "natural": "^0.4.0", "nodemailer": "^2.6.4", + "parse-duration": "^0.1.1", "passport": "^0.3.2", "passport-facebook": "^2.1.1", "passport-local": "^1.0.0", diff --git a/routes/api/asset/index.js b/routes/api/asset/index.js index 18fd0b7ec..0c740d32f 100644 --- a/routes/api/asset/index.js +++ b/routes/api/asset/index.js @@ -11,17 +11,20 @@ router.get('/', (req, res, next) => { limit = 20, skip = 0, sort = 'asc', - field = 'created_at' + field = 'created_at', + search = '' } = req.query; // Find all the assets. Promise.all([ Asset - .find({}) + .search(search) .sort({[field]: (sort === 'asc') ? 1 : -1}) .skip(skip) .limit(limit), - Asset.count() + Asset + .search(search) + .count() ]) .then(([result, count]) => { diff --git a/tests/mongoose.js b/tests/mongoose.js index 9d3ba1195..74a0b04f7 100644 --- a/tests/mongoose.js +++ b/tests/mongoose.js @@ -2,8 +2,8 @@ const mongoose = require('../mongoose'); beforeEach(function (done) { function clearDB() { - for (let i in mongoose.connection.collections) { - mongoose.connection.collections[i].remove(function() {}); + for (let collection in mongoose.connection.collections) { + mongoose.connection.collections[collection].remove(function() {}); } return done(); } diff --git a/tests/routes/api/assets/index.js b/tests/routes/api/assets/index.js index fb67b7b48..667bec0ed 100644 --- a/tests/routes/api/assets/index.js +++ b/tests/routes/api/assets/index.js @@ -1,9 +1,84 @@ +const passport = require('../../../passport'); + +const app = require('../../../../app'); +const chai = require('chai'); +const expect = chai.expect; + +// Setup chai. +chai.should(); +chai.use(require('chai-http')); + +const Asset = require('../../../../models/asset'); + describe('/assets', () => { + beforeEach(() => { + return Asset.create([ + { + url: 'https://coralproject.net/news/asset1', + title: 'Asset 1', + description: 'term1' + }, + { + url: 'https://coralproject.net/news/asset2', + title: 'Asset 2', + description: 'term2' + } + ]); + }); + describe('GET', () => { - it('should return assets that we search for'); - it('should not return assets that we do not search for'); + it('should return all assets without a search query', () => { + return chai.request(app) + .get('/api/v1/asset') + .set(passport.inject({roles: ['admin']})) + .then((res) => { + const body = res.body; + + expect(body).to.have.property('count', 2); + expect(body).to.have.property('result'); + + const assets = body.result; + + expect(assets).to.have.length(2); + }); + }); + + it('should return assets that we search for', () => { + return chai.request(app) + .get('/api/v1/asset?search=term2') + .set(passport.inject({roles: ['admin']})) + .then((res) => { + const body = res.body; + + expect(body).to.have.property('count', 1); + expect(body).to.have.property('result'); + + const assets = body.result; + + expect(assets).to.have.length(1); + + const asset = assets[0]; + + expect(asset).to.have.property('url', 'https://coralproject.net/news/asset2'); + expect(asset).to.have.property('title', 'Asset 2'); + }); + }); + + it('should not return assets that we do not search for', () => { + return chai.request(app) + .get('/api/v1/asset?search=term3') + .set(passport.inject({roles: ['admin']})) + .then((res) => { + const body = res.body; + + expect(body).to.have.property('count', 0); + expect(body).to.have.property('result'); + + expect(body.result).to.be.empty; + }); + }); });