Execute Grunt task using NodeJS cluster - node.js

Question

I have a Grunt task and currently I am utilising AsyncJS to run it. AsyncJS worked well but still I feel like it can be more powerful if I can utilise NodeJS cluster to run it. I have checked out Grunt Parallel and Grunt Concurrent and it is not much different to what I doing in my Grunt task. Any suggestions on utilising NodeJS cluster module to speed up Task execution.
Currently I am doing like this
var queue = async.queue(task, function(task, cb){
// Process task with PhantomJS and then
cb();
}, require('os').cpus().length);
async.each(htmlPages, function(val, cb) {
queue.push(val, function() {
cb();
});
}, function() {
console.log('Completed');
done();
});
How can I make this work with NodeJS cluster?

Answer 1

One way to do it is to spawn the number of workers that you want using the cluster module. Then send messages to them when you want to start them working on something.
Below is code that initialises os.cpus().length workers and a queue that sends the work to them. It then pushes everything in htmlPages to that queue, waits for it to finish and then finally kills all the workers.
var os = require('os');
var async = require('async');
var cluster = require('cluster');
if (cluster.isWorker) {
process.on('message', function(msg) {
// Do the Phantom JS stuff
process.send(theResult);
});
}
if (cluster.isMaster) {
var workers = os.cpus().map(function () {
return cluster.fork();
});
var queue = async.queue(function (msg, cb) {
var worker = workers.pop();
worker.once('message', function (msg) {
workers.push(worker);
cb(null, msg);
});
worker.send(msg);
}, workers.length);
async.each(htmlPages, queue.push.bind(queue), function (err) {
if (err) { throw err; }
workers.forEach(function (worker) {
worker.kill();
});
console.log('Completed');
});
}