talk/bin/cli-jobs

#!/usr/bin/env node

/**
 * Module dependencies.
 */

const program = require('./commander');
const scraper = require('../services/scraper');
const mailer = require('../services/mailer');
const util = require('./util');
const mongoose = require('../services/mongoose');
const kue = require('../services/kue');

util.onshutdown([
  () => mongoose.disconnect(),
]);

/**
 * Starts the job processor.
 */
function processJobs() {

  // The scraper only needs to shutdown when the scraper has actually been
  // started.
  util.onshutdown([
    () => kue.Task.shutdown()
  ]);

  // Start the scraper processor.
  scraper.process();

  // Start the mail processor.
  mailer.process();
}

/**
 * Removes a single job.
 * @param  {Object} job the job to be removed
 * @return {Promise}
 */
function removeJob(job) {
  return new Promise((resolve, reject) => job.remove((err) => {
    if (err) {
      return reject(err);
    }

    return resolve(job);
  }));
}

/**
 * Get the top n jobs with a specific state.
 * @param  {String} [state='complete'] state to list jobs by
 * @param  {Number} limit              limit of jobs to load
 * @return {Promise}
 */
function rangeJobsByState(state, limit) {
  return new Promise((resolve, reject) => {
    kue.Job.rangeByState(state, 0, limit, 'asc', (err, jobs) => {
      if (err) {
        return reject(err);
      }

      resolve(jobs);
    });
  });
}

async function getJobBatch(n, includeStuck) {
  let jobs = [];

  jobs = await rangeJobsByState('complete', n);

  if (includeStuck) {
    jobs = jobs.concat(await rangeJobsByState('failed', n));
  }

  return jobs;
}

/**
 * Cleans up the jobs that are in the queue.
 */
async function cleanupJobs(options) {

  // The scraper only needs to shutdown when the scraper has actually been
  // started.
  util.onshutdown([
    () => kue.Task.shutdown()
  ]);

  const n = 100;

  try {

    // Connect to redis by establishing a queue.
    kue.Task.connect();

    let jobCount = 0;
    let jobs = await getJobBatch(n, options.stuck);

    while (jobs.length > 0) {

      // Remove all the jobs.
      await Promise.all(jobs.map((job) => removeJob(job)));

      jobCount += jobs.length;

      // Get the next batch of jobs.
      jobs = await getJobBatch(n, options.stuck);
    }

    util.shutdown();
    console.log(`Removed ${jobCount} jobs`);
  } catch (err) {
    console.error(err);
    util.shutdown(1);
  }
}

//==============================================================================
// Setting up the program command line arguments.
//==============================================================================

program
  .command('process')
  .description('starts job processing')
  .action(processJobs);

program
  .command('cleanup')
  .option('-s, --stuck', 'cleans up jobs that have been stuck', false)
  .description('cleans up inactive jobs')
  .action(cleanupJobs);

program.parse(process.argv);

// If there is no command listed, output help.
if (process.argv.length <= 2) {
  program.outputHelp();
  util.shutdown();
}