I am stuck here due to a simple event related issue. Here is the issue:
I have created a cluster using cluster.js and forked server.js from
cluster.js.
I have put a timer from cluster.js and after every 1 min I am
triggering an event 'testTimer'. I have used a event file to do
it.
I am trying to capture this event 'testTimer' from the child
process using the same file I have imported into server.js and doing
a .on('testTimer', callback)
However, the events are not captured in any of the processes. I have tried making the event global and assign the event globally to a symbol but was unable to get it work/capture event as well.
Here is the codes:
cluster.js (child process creator)
...require > events.js...
... create cluster logic...
setInterval(function () {
evt.emit('testTimer', {tester: 'test'});
evt.tester();
}, 1000);
server.js (child process)
...require > events.js...
evt.on('testTimer', function (data) {
console.log('Starting Sync ', data);
});
events.js (common file for events)
var util = require("util");
var EventEmitter = require("events").EventEmitter;
function test () {
EventEmitter.call(this);
}
test.prototype.tester = function (){
this.emit('testTimer', {missed: 'this'})
}
util.inherits(test, EventEmitter);
module.exports = test;
EventEmitter instances can't reach beyond the bounds of a process. If you want to communicate between parent and children, use worker.send():
// cluster.js
setInterval(function () {
for (const id in cluster.workers) {
cluster.workers[id].send({ type : 'testTimer', data : { tester : 'test' }});
}
}, 1000);
// server.js
process.on('message', function(message) {
if (message.type === 'testTimer') {
console.log('Starting Sync ', message.data);
}
})
Related
How can I manage multiple concurrent child processes that have been forked?
In this example, start_child() can be invoked multiple times, and each invocation can run indefinitely. When forking an arbitrary number of child processes like this, how can I communicate with / address each individual child process? Let's say I have 3 forked child processes running, and they keep running indefinitely, but I want to kill (or send a message to) child process number 2. How would I do that?
If stop_child() is invoked, it kills all of the currently running child processes. How do I refactor this code so that I can call stop_child() on an individual child process?
let app = require('express')();
let server = require('http').Server(app);
let io = require('socket.io')(server);
let fork = require('child_process').fork;
server.listen(80);
app.get('/', function(request, response) {
response.sendFile(__dirname + '/index.html');
});
io.on('connection', function(socket) {
socket.on('start_child', function () {
start_child();
});
socket.on('stop_child', function () {
child.kill();
}
}
function start_child() {
child = fork('./child_script.js');
//conditional logic to either run
//start_child() again or let it end
}
UPDATE
I tried this, based on some of the comments. But if I launch 3 processes and then call, for example, child[0].kill(), I get an error: Cannot read property 'kill' of undefined. I'm guessing my problem is that I'm not correctly passing the i variable to the io.on() call:
let app = require('express')();
let server = require('http').Server(app);
let io = require('socket.io')(server);
let fork = require('child_process').fork;
let i = 0;
let child = [];
server.listen(80);
app.get('/', function(request, response) {
response.sendFile(__dirname + '/index.html');
});
io.on('connection', function(socket) {
socket.on('start_child', function (i) {
start_child(i++);
});
//this is hard coded just for testing
socket.on('stop_child', function () {
child[0].kill();
}
}
function start_child(i) {
child[i] = fork('./child_script.js');
//conditional logic to either run
//start_child() again or let it end
}
UPDATE #2
Okay, I figured out that I need to send the incrementing variable from the client side, passed through an object coming from the emit call. Now when I call child[0].kill() there is no error. The problem is that the child process is not killed:
server.js
let app = require('express')();
let server = require('http').Server(app);
let io = require('socket.io')(server);
let fork = require('child_process').fork;
server.listen(80);
app.get('/', function(request, response) {
response.sendFile(__dirname + '/index.html');
});
io.on('connection', function(socket) {
socket.on('start_child', function (count) {
let num = count.count;
start_child(num);
});
//this is hard coded just for testing
socket.on('stop_child', function (count) {
let num = count.count;
child[num].kill();
}
}
function start_child(num) {
child[num] = fork('./child_script.js');
//conditional logic to either run
//start_child() again or let it end
}
index.html
$(function () {
let socket = io();
let i = 0;
$('#start').on('click', function () {
socket.emit('start_child', {"count": i++});
});
$('#stop').on('click', function () {
//the count is hard coded here just for testing purposes
socket.emit('stop_child', {"count": 0});
});
});
FINAL UPDATE - WITH RESOLUTION
Resolution #2 (right above this) is actually the solution. The problem I was having after that (where the child.kill() call didn't seem to do anything) was caused by a piece of the code that I had left out (in the code comment: 'conditional logic to either run start_child() again or let it end').
This is what was in there:
if (condition) {
setTimeout(start_child, 5000);
} else {
console.log('this child process has ended');
}
And this is what I changed it to (basically, I just had to pass the incrementing variable to the start_child() function so that it would have the same place in the child array when it restarted):
if (condition) {
setTimeout(function() {
start_child(num)
}, 5000);
} else {
console.log('this child process has ended');
}
fork() returns a ChildProcess object which has methods and events on that object for interprocess communication with that process. So you have to save each ChildProcess object from when you call start_child() and then use the appropriate object in order to communicate with the other process.
You can see events and methods of the ChildProcess object here. There are also numerous code examples.
In order to send message to the child process, use the following example :
parent.js
**
const { fork } = require('child_process');
const forked = fork('child.js');
forked.on('message', (msg) => {
console.log('Message from child', msg);
});
forked.send({ hello: 'world' });
**
child.js
process.on('message', (msg) => {
console.log('Message from parent:', msg);
});
let counter = 0;
setInterval(() => {
process.send({ counter: counter++ });
}, 1000);
For further information refer this article :
https://medium.freecodecamp.com/node-js-child-processes-everything-you-need-to-know-e69498fe970a
I have a node.js server, which will print out some message in the console and then start the server.
I am creating a automation test by using tap to check the message in the console.log and check if server is started, i.e. there is a PID generated.
I tried 2 different methods -child_process.exec and child_process.spawn
1. Use child_process.exec with a call back function.
This does not work as the server is long running and will not even
go to the call back, so I cannot even check for any stdout.
Then I use child_process.exec without call back, this solves the
first issue where I can now get the message back from stdout.
The second issue is that the test will hang since the server is long running and will not terminate by itself.
code snippet:
var exec = require('child_process').exec;
tap.test('test server start', function(t) {
childProcess= exec('node',['server']);
console.log('[exec] childProcess.pid: ', childProcess.pid);
t.notEqual(childProcess.pid, undefined);
childProcess.stdout.on('data', function (data) {
console.log('[exec] stdout: ', data.toString());
t.match(data.toString(), "Example app listening at http://:::3000");
t.end();
childProcess.kill('SIGTERM');
});
childProcess.stderr.on('data', function (data) {
console.log('[exec] stderr: ', data.toString());
});
childProcess.on('close', function (code) {
if (code!=null)
console.log('child process exited with code '+ code);
});
});
use child_process.spawn -code snippet
var spawn = require('child_process').spawn;
tap.test('test server start', function(t) {
childProcess= spawn('node',['server']);
console.log('[spawn] childProcess.pid: ', childProcess.pid);
t.notEqual(childProcess.pid, undefined);
childProcess.stdout.on('data', function (data) {
console.log('[spawn] stdout: ', data.toString());
t.match(data.toString(), "Example app listening at http://:::3000");
t.end();
childProcess.kill('SIGTERM');
});
childProcess.stderr.on('data', function (data) {
console.log('[spawn] stderr: ', data.toString());
});
childProcess.on('close', function (code) {
if (code!=null)
console.log('child process exited with code '+ code);
});
});
In both 1 & 2, the test will hang since the server is long running,
I need to use child_process.kill() to terminate the test
Is there a better method to achieve this?
Thanks in advance for any improvements.
Well, I think that you can check if the server is alive in a different way (without spawning a new process).
For example, you can start your server waiting for connections:
const net = require('net');
var connections = {};
var server = net.createServer(function(conn) { });
server.listen(3333);
server.on('connection',function(conn) {
var key = conn.remoteAddress + ':' + conn.remotePort;
connections[key] = conn;
conn.on('close',function() {
delete connections[key];
});
});
Then, connect some clients (or just one) to the server:
var connected = 0;
for (var i = 0;i < 10;i++) {
var client = net.connect(3333);
client.on('connect',function() {
connected++;
console.log(connected);
});
}
So, if you are be able to connect to the server, then your server is alive.
And finally, when you want to close the server, just create a new function like this one:
var destroy = function ()
{
server.close(function() {
console.log('ok');
});
for (var key in connections) {
connections[key].destroy();
}
}
Call it for example after 10 successful connections to the server. Inside the for loop:
for (var i = 0; i < 10; i++) {
var client = net.connect(3333);
client.on('connect',function() {
connected++;
if (connected === 10) {
destroy();
}
});
}
This is a very basic example, but I think that it's enough to understand another way to do what you want to do.
I.
I'm using node-cron module for scheduling tasks in Node.js application. I also want run the application in several processes using core cluster module.
Running application in several processes ends up in scheduled tasks execution in each process (e.g. if task was to send an email the email would be sent multiple times).
What are the best practices/possible ways of running cron job along with cluster module? Should I create some separate process which will handle only cron job and do not accept any requests. If yes, how can I do that in a right way?
If are using PM2,
You can use an environment variable provided by PM2 itself called NODE_APP_INSTANCE which requires PM2 2.5 or greater.
NODE_APP_INSTANCE environment variable can be used to determine difference between process, for example you may want to run a cronjob only on one process, you can just do this
if(process.env.NODE_APP_INSTANCE == 0) {
//schedule your cron job here since this part will be executed for only one cluster
}
,
Since two processes can never have the same number.
More Info on PM2 official doc here.
After some research I ended up with "Distributed locks using Redis" solution.
There is node module for that: node-redis-warlock.
Hope this answer will be useful for someone else.
UPDATE. Minimal sample code:
var Warlock = require('node-redis-warlock'),
redis = require('redis');
// Establish a redis client
redis = redis.createClient();
// and pass it to warlock
var warlock = new Warlock(redis);
function executeOnce (key, callback) {
warlock.lock(key, 20000, function(err, unlock){
if (err) {
// Something went wrong and we weren't able to set a lock
return;
}
if (typeof unlock === 'function') {
setTimeout(function() {
callback(unlock);
}, 1000);
}
});
}
// Executes call back only once
executeOnce('every-three-hours-lock', function(unlock) {
// Do here any stuff that should be done only once...
unlock();
});
UPDATE 2. More detailed example:
const CronJob = require('cron').CronJob;
const Warlock = require('node-redis-warlock');
const redis = require('redis').createClient();
const warlock = new Warlock(redis);
const async = require('async');
function executeOnce (key, callback) {
warlock.lock(key, 20000, function(err, unlock) {
if (err) {
// Something went wrong and we weren't able to set a lock
return;
}
if (typeof unlock === 'function') {
setTimeout(function() {
callback(unlock);
}, 1000);
}
});
}
function everyMinuteJobTasks (unlock) {
async.parallel([
sendEmailNotifications,
updateSomething,
// etc...
],
(err) => {
if (err) {
logger.error(err);
}
unlock();
});
}
let everyMinuteJob = new CronJob({
cronTime: '*/1 * * * *',
onTick: function () {
executeOnce('every-minute-lock', everyMinuteJobTasks);
},
start: true,
runOnInit: true
});
/* Actual tasks */
let sendEmailNotifications = function(done) {
// Do stuff here
// Call done() when finished or call done(err) if error occurred
}
let updateSomething = function(done) {
// Do stuff here
// Call done() when finished or call done(err) if error occurred
}
// etc...
I think you can use the node cluster module, and there you can write your code to run in the master cluster only
const cluster = require('cluster');
if (cluster.isMaster) {
// Write your code which you want to execute in the master cluster only
}
This is a node way to handle cluster, of course, you can use any tool like pm2 to handle this.
I actually do not like the redis approach that is also used in the cron-cluster npm plugin, because I do not want to have that redis server running on my maschine and maintain it, too.
I would like to discuss this approach with you:
Pro: we do not need to use redis
Con: cron jobs are always running on the same worker
I use the message passing only for this, if you use it for other things, you want to pass the information that
if (cluster.isMaster) {
// Count the machine's CPUs
var cpuCount = require('os').cpus().length;;
// Create a worker for each CPU
for (var i = 0; i < cpuCount; i += 1) {
cluster.fork();
}
cluster.on('fork', (worker) => {
console.log("cluster forking new worker", worker.id);
});
// have a mainWorker that does the cron jobs.
var mainWorkerId = null;
cluster.on('listening', (worker, address) => {
console.log("cluster listening new worker", worker.id);
if(null === mainWorkerId) {
console.log("Making worker " + worker.id + " to main worker");
mainWorkerId = worker.id;
worker.send({order: "startCron"});
}
});
// Listen for dying workers if the mainWorker dies, make a new mainWorker
cluster.on('exit', function (worker, code, signal) {
console.log('Worker %d died :(', worker.id);
if(worker.id === mainWorkerId) {
console.log("Main Worker is dead...");
mainWorkerId = null;
}
console.trace("I am here");
console.log(worker);
console.log(code);
console.log(signal);
cluster.fork();
});
// Code to run if we're in a worker process
} else {
// other code like setup app and stuff
var doCron = function() {
// setup cron jobs...
}
// Receive messages from the master process.
process.on('message', function(msg) {
console.log('Worker ' + process.pid + ' received message from master.', message);
if(message.order == "startCron") {
doCron();
}
});
}
I also have a problem with cluster module and finally i found sample way to solve problem.
Let master cluster execute cronJob.
My project use Kue to manage jobs. When cronJob run i get a list of jobs.
index.js
global.cluster = require('cluster');
if (cluster.isMaster) {
const cpuCount = require('os').cpus().length;
for (let i = 0; i < cpuCount; i += 1) {
cluster.fork();
}
} else {
// start your express server here
require('./server')
}
cluster.on('exit', worker => {
logger.warn('Worker %d died :(', worker.id);
cluster.fork();
});
cron.js
const cron = require('cron').CronJob;
const job = new cron('* * * * *', async () => {
if (cluster.isMaster) {
console.log('cron trigger');
}
});
job.start();
Hope this help.
I have a Grunt task and currently I am utilising AsyncJS to run it. AsyncJS worked well but still I feel like it can be more powerful if I can utilise NodeJS cluster to run it. I have checked out Grunt Parallel and Grunt Concurrent and it is not much different to what I doing in my Grunt task. Any suggestions on utilising NodeJS cluster module to speed up Task execution.
Currently I am doing like this
var queue = async.queue(task, function(task, cb){
// Process task with PhantomJS and then
cb();
}, require('os').cpus().length);
async.each(htmlPages, function(val, cb) {
queue.push(val, function() {
cb();
});
}, function() {
console.log('Completed');
done();
});
How can I make this work with NodeJS cluster?
One way to do it is to spawn the number of workers that you want using the cluster module. Then send messages to them when you want to start them working on something.
Below is code that initialises os.cpus().length workers and a queue that sends the work to them. It then pushes everything in htmlPages to that queue, waits for it to finish and then finally kills all the workers.
var os = require('os');
var async = require('async');
var cluster = require('cluster');
if (cluster.isWorker) {
process.on('message', function(msg) {
// Do the Phantom JS stuff
process.send(theResult);
});
}
if (cluster.isMaster) {
var workers = os.cpus().map(function () {
return cluster.fork();
});
var queue = async.queue(function (msg, cb) {
var worker = workers.pop();
worker.once('message', function (msg) {
workers.push(worker);
cb(null, msg);
});
worker.send(msg);
}, workers.length);
async.each(htmlPages, queue.push.bind(queue), function (err) {
if (err) { throw err; }
workers.forEach(function (worker) {
worker.kill();
});
console.log('Completed');
});
}
Grunt notify: https://github.com/dylang/grunt-notify is great. However, it seems a bit limited. As far as I can tell all my messages need to be pre-generated. So the first question is how can I generate notifications?
Next, It seems that grunt notify triggers based on error or success of some task. I guess based on std in/out/err? The problem where this breaks down is if some task doesn't use these. grunt compass doesn't use stderr if there are compile errors. So how can I run grunt notify when it has an error? This then leads to the next question. How can I grab the console output or stderr from a grunt task?
First of all, use growl. It is easy and flexible to use. To install growl:
npm install growl --save-dev
Then you need to hook into the stderr/out stream of the process. This way you can create a notification each time a new message arrives at the stderr/out stream.
This is what I've created. I've made a CommonJs module which adds hooks to:
grunt.fail.warn(), grunt.fail.fatal()
grunt.log.warn(), grunt.log.error()
grunt.warn()
process.stderr.write()
process.stdout.write() (error lines)
(child) process.stderr.write()
(child) process.stdout.write() (error lines)
It works more or less, but it may need some tweaking.
tasks/lib/notify.js
(function (module) {
var grunt = require('grunt'),
growl = require('growl'),
Buffer = require('buffer').Buffer;
function notify(obj, title) {
if (obj) {
var message = Buffer.isBuffer(obj) ? obj.toString() : (obj.message || obj);
var msg = grunt.log.uncolor(message);
if (msg.length > 0) {
growl(msg, {
title: title,
image: 'Console'
});
}
}
}
// add a hook to grunt.fail.warn(), grunt.fail.fatal()
['warn', 'fatal'].forEach(function (level) {
grunt.util.hooker.hook(grunt.fail, level, function(obj) {
notify(obj);
});
});
// add a hook to grunt.log.warn(), grunt.log.error()
['warn', 'error'].forEach(function (level) {
grunt.util.hooker.hook(grunt.log, level, function(obj) {
notify(obj, level);
});
});
// add a hook to grunt.warn()
grunt.util.hooker.hook(grunt, 'warn', function(obj) {
notify(obj, 'warn');
});
// add a hook to process.stderr.write()
grunt.util.hooker.hook(process.stderr, 'write', function(obj) {
var messages = grunt.log.uncolor((Buffer.isBuffer(obj) ? obj.toString() : (obj.message || obj))).split('\n');
messages.forEach(function (message) {
notify(message, 'stderr');
});
});
// add a hook to process.stdout.write() (only error lines)
grunt.util.hooker.hook(process.stdout, 'write', function(obj) {
var messages = grunt.log.uncolor((Buffer.isBuffer(obj) ? obj.toString() : (obj.message || obj))).split('\n');
messages.forEach(function (message) {
if (message && message.indexOf('error ') > -1) {
notify(message, 'stdout');
}
});
});
// add a hook to child process stdout/stderr write() (only error lines)
grunt.util.hooker.hook(grunt.util, 'spawn', {
post: function(child) {
child.stderr.on('data', function (data) {
var messages = grunt.log.uncolor(data.toString()).split('\n');
messages.forEach(function (message) {
notify(message, 'stderr');
});
});
child.stdout.on('data', function (data) {
var messages = grunt.log.uncolor(data.toString()).split('\n');
messages.forEach(function (message) {
if (message && message.indexOf('error ') > -1) {
notify(message, 'stdout');
}
});
});
}
});
}) (module);
Then you need to include it in your Gruntfile.js with a require statement:
module.exports = function (grunt) {
grunt.initConfig({
...
});
require('./tasks/lib/notify');
...
};
PS I've placed the file in tasks/lib/notify.js, but feel free to place it somewhere else.