#!/usr/bin/env node /** * Module dependencies. */ const util = require('./util'); const program = require('commander'); const scraper = require('../services/scraper'); const mailer = require('../services/mailer'); const mongoose = require('../services/mongoose'); const kue = require('../services/kue'); util.onshutdown([() => mongoose.disconnect()]); /** * Starts the job processor. */ function processJobs() { // The scraper only needs to shutdown when the scraper has actually been // started. util.onshutdown([() => kue.Task.shutdown()]); // Start the scraper processor. scraper.process(); // Start the mail processor. mailer.process(); } /** * Removes a single job. * @param {Object} job the job to be removed * @return {Promise} */ function removeJob(job) { return new Promise((resolve, reject) => job.remove(err => { if (err) { return reject(err); } return resolve(job); }) ); } /** * Get the top n jobs with a specific state. * @param {String} [state='complete'] state to list jobs by * @param {Number} limit limit of jobs to load * @return {Promise} */ function rangeJobsByState(state, limit) { return new Promise((resolve, reject) => { kue.Job.rangeByState(state, 0, limit, 'asc', (err, jobs) => { if (err) { return reject(err); } resolve(jobs); }); }); } async function getJobBatch(n, includeStuck) { let jobs = []; jobs = await rangeJobsByState('complete', n); if (includeStuck) { jobs = jobs.concat(await rangeJobsByState('failed', n)); } return jobs; } /** * Cleans up the jobs that are in the queue. */ async function cleanupJobs(options) { // The scraper only needs to shutdown when the scraper has actually been // started. util.onshutdown([() => kue.Task.shutdown()]); const n = 100; try { // Connect to redis by establishing a queue. kue.Task.connect(); let jobCount = 0; let jobs = await getJobBatch(n, options.stuck); while (jobs.length > 0) { // Remove all the jobs. await Promise.all(jobs.map(job => removeJob(job))); jobCount += jobs.length; // Get the next batch of jobs. jobs = await getJobBatch(n, options.stuck); } util.shutdown(); console.log(`Removed ${jobCount} jobs`); } catch (err) { console.error(err); util.shutdown(1); } } //============================================================================== // Setting up the program command line arguments. //============================================================================== program .command('process') .description('starts job processing') .action(processJobs); program .command('cleanup') .option('-s, --stuck', 'cleans up jobs that have been stuck', false) .description('cleans up inactive jobs') .action(cleanupJobs); program.parse(process.argv); // If there is no command listed, output help. if (process.argv.length <= 2) { program.outputHelp(); util.shutdown(); }