Added asset searching

bin/cli-assets

@@ -12,9 +12,11 @@ process.env.DEBUG = process.env.TALK_DEBUG;
 
 const program = require('commander');
 const pkg = require('../package.json');
+const parseDuration = require('parse-duration');
 const Table = require('cli-table');
 const Asset = require('../models/asset');
 const mongoose = require('../mongoose');
+const scraper = require('../services/scraper');
 const util = require('../util');
 
 // Register the shutdown criteria.
@@ -55,6 +57,37 @@ function listAssets() {
     });
 }
 
+function refreshAssets(ageString) {
+  const now = new Date().getTime();
+  const ageMs = parseDuration(ageString);
+  const age = new Date(now - ageMs);
+
+  Asset.find({
+    $or: [
+      {
+        scraped: {
+          $lte: age
+        }
+      },
+      {
+        scraped: null
+      }
+    ]
+  })
+
+  // Queue all the assets for scraping.
+  .then((assets) => Promise.all(assets.map(scraper.create)))
+
+  .then(() => {
+    console.log('Assets were queued to be scraped');
+    util.shutdown();
+  })
+  .catch((err) => {
+    console.error(err);
+    util.shutdown(1);
+  });
+}
+
 //==============================================================================
 // Setting up the program command line arguments.
 //==============================================================================
@@ -67,6 +100,11 @@ program
   .description('list all the assets in the database')
   .action(listAssets);
 
+program
+  .command('refresh <age>')
+  .description('queues the assets that exceed the age requested')
+  .action(refreshAssets);
+
 program.parse(process.argv);
 
 // If there is no command listed, output help.

bin/cli-jobs

@@ -14,11 +14,15 @@ const program = require('commander');
 const scraper = require('../services/scraper');
 const util = require('../util');
 const mongoose = require('../mongoose');
+const kue = require('../kue');
 
 util.onshutdown([
   () => mongoose.disconnect()
 ]);
 
+/**
+ * Starts the job processor.
+ */
 function processJobs() {
 
   // Start the processor.
@@ -31,6 +35,65 @@ function processJobs() {
   ]);
 }
 
+/**
+ * Removes a single job.
+ * @param  {Object} job the job to be removed
+ * @return {Promise}
+ */
+function removeJob(job) {
+  return new Promise((resolve, reject) => job.remove((err) => {
+    if (err) {
+      return reject(err);
+    }
+
+    return resolve(job);
+  }));
+}
+
+/**
+ * Removes the jobs passed in and returns a promise.
+ * @param  {Array} jobs array of jobs
+ * @return {Promise}
+ */
+function removeJobs(jobs) {
+  return Promise.all(jobs.map(removeJob));
+}
+
+/**
+ * Get the top n jobs with a specific state.
+ * @param  {String} [state='complete'] state to list jobs by
+ * @param  {Number} limit              limit of jobs to load
+ * @return {Promise}
+ */
+function rangeJobsByState(state = 'complete', limit) {
+  return new Promise((resolve, reject) => {
+    kue.Job.rangeByState(state, 0, limit, 'asc', (err, jobs) => {
+      if (err) {
+        return reject(err);
+      }
+
+      resolve(jobs);
+    });
+  });
+}
+
+/**
+ * Cleans up the jobs that are in the queue.
+ */
+function cleanupJobs(options) {
+  const n = 100;
+
+  Promise.all([
+    rangeJobsByState('complete', n),
+    options.stuck ? rangeJobsByState('failed', n) : false
+  ])
+  .then((joblists) => joblists.filter((jobs) => jobs).map(removeJobs))
+  .then(() => {
+    util.shutdown();
+    console.log('Removed old jobs');
+  });
+}
+
 //==============================================================================
 // Setting up the program command line arguments.
 //==============================================================================
@@ -40,6 +103,12 @@ program
   .description('starts job processing')
   .action(processJobs);
 
+program
+  .command('cleanup')
+  .option('-s, --stuck', 'cleans up jobs that have been stuck', false)
+  .description('cleans up inactive jobs')
+  .action(cleanupJobs);
+
 program.parse(process.argv);
 
 // If there is no command listed, output help.

models/asset.js

@@ -38,21 +38,32 @@ const AssetSchema = new Schema({
   }
 });
 
+AssetSchema.index({
+  title: 'text',
+  url: 'text',
+  description: 'text',
+  section: 'text',
+  subsection: 'text',
+  author: 'text'
+}, {
+  background: true
+});
+
 /**
  * Search for assets. Currently only returns all.
-*/
+ */
 AssetSchema.statics.search = (query) => Asset.find(query);
 
 /**
  * Finds an asset by its id.
  * @param {String} id  identifier of the asset (uuid).
-*/
+ */
 AssetSchema.statics.findById = (id) => Asset.findOne({id});
 
 /**
  * Finds a asset by its url.
  * @param {String} url  identifier of the asset (uuid).
-*/
+ */
 AssetSchema.statics.findByUrl = (url) => Asset.findOne({url});
 
 /**
@@ -65,7 +76,8 @@ AssetSchema.statics.findByUrl = (url) => Asset.findOne({url});
  * is not possible with the mongoose driver.
  *
  * @param {String} url  identifier of the asset (uuid).
-*/
+ * @return {Promise}
+ */
 AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {url}, {
 
   // Ensure that if it's new, we return the new object created.
@@ -78,6 +90,18 @@ AssetSchema.statics.findOrCreateByUrl = (url) => Asset.findOneAndUpdate({url}, {
   setDefaultsOnInsert: true
 });
 
+/**
+ * Finds assets matching keywords on the model. If `value` is an empty string,
+ * then it will not even perform a text search query.
+ * @param  {String} value string to search by.
+ * @return {Promise}
+ */
+AssetSchema.statics.search = (value) => value.length === 0 ? Asset.find({}) : Asset.find({
+  $text: {
+    $search: value
+  }
+});
+
 const Asset = mongoose.model('Asset', AssetSchema);
 
 module.exports = Asset;

package.json

@@ -16,8 +16,13 @@
   "config": {
     "pre-git": {
       "commit-msg": [],
-      "pre-commit": ["npm run lint", "npm test"],
-      "pre-push": ["npm test"],
+      "pre-commit": [
+        "npm run lint",
+        "npm test"
+      ],
+      "pre-push": [
+        "npm test"
+      ],
       "post-commit": [],
       "post-merge": []
     }
@@ -26,7 +31,12 @@
     "type": "git",
     "url": "git+https://github.com/coralproject/talk.git"
   },
-  "keywords": ["talk", "coral", "coralproject", "ask"],
+  "keywords": [
+    "talk",
+    "coral",
+    "coralproject",
+    "ask"
+  ],
   "author": "",
   "license": "Apache-2.0",
   "bugs": {
@@ -52,6 +62,7 @@
     "morgan": "^1.7.0",
     "natural": "^0.4.0",
     "nodemailer": "^2.6.4",
+    "parse-duration": "^0.1.1",
     "passport": "^0.3.2",
     "passport-facebook": "^2.1.1",
     "passport-local": "^1.0.0",

routes/api/asset/index.js

@@ -11,17 +11,20 @@ router.get('/', (req, res, next) => {
     limit = 20,
     skip = 0,
     sort = 'asc',
-    field = 'created_at'
+    field = 'created_at',
+    search = ''
   } = req.query;
 
   // Find all the assets.
   Promise.all([
     Asset
-      .find({})
+      .search(search)
       .sort({[field]: (sort === 'asc') ? 1 : -1})
       .skip(skip)
       .limit(limit),
-    Asset.count()
+    Asset
+      .search(search)
+      .count()
   ])
   .then(([result, count]) => {
 

tests/mongoose.js

@@ -2,8 +2,8 @@ const mongoose = require('../mongoose');
 
 beforeEach(function (done) {
   function clearDB() {
-    for (let i in mongoose.connection.collections) {
-      mongoose.connection.collections[i].remove(function() {});
+    for (let collection in mongoose.connection.collections) {
+      mongoose.connection.collections[collection].remove(function() {});
     }
     return done();
   }

tests/routes/api/assets/index.js

@@ -1,9 +1,84 @@
+const passport = require('../../../passport');
+
+const app = require('../../../../app');
+const chai = require('chai');
+const expect = chai.expect;
+
+// Setup chai.
+chai.should();
+chai.use(require('chai-http'));
+
+const Asset = require('../../../../models/asset');
+
 describe('/assets', () => {
 
+  beforeEach(() => {
+    return Asset.create([
+      {
+        url: 'https://coralproject.net/news/asset1',
+        title: 'Asset 1',
+        description: 'term1'
+      },
+      {
+        url: 'https://coralproject.net/news/asset2',
+        title: 'Asset 2',
+        description: 'term2'
+      }
+    ]);
+  });
+
   describe('GET', () => {
 
-    it('should return assets that we search for');
-    it('should not return assets that we do not search for');
+    it('should return all assets without a search query', () => {
+      return chai.request(app)
+        .get('/api/v1/asset')
+        .set(passport.inject({roles: ['admin']}))
+        .then((res) => {
+          const body = res.body;
+
+          expect(body).to.have.property('count', 2);
+          expect(body).to.have.property('result');
+
+          const assets = body.result;
+
+          expect(assets).to.have.length(2);
+        });
+    });
+
+    it('should return assets that we search for', () => {
+      return chai.request(app)
+        .get('/api/v1/asset?search=term2')
+        .set(passport.inject({roles: ['admin']}))
+        .then((res) => {
+          const body = res.body;
+
+          expect(body).to.have.property('count', 1);
+          expect(body).to.have.property('result');
+
+          const assets = body.result;
+
+          expect(assets).to.have.length(1);
+
+          const asset = assets[0];
+
+          expect(asset).to.have.property('url', 'https://coralproject.net/news/asset2');
+          expect(asset).to.have.property('title', 'Asset 2');
+        });
+    });
+
+    it('should not return assets that we do not search for', () => {
+      return chai.request(app)
+        .get('/api/v1/asset?search=term3')
+        .set(passport.inject({roles: ['admin']}))
+        .then((res) => {
+          const body = res.body;
+
+          expect(body).to.have.property('count', 0);
+          expect(body).to.have.property('result');
+
+          expect(body.result).to.be.empty;
+        });
+    });
 
   });