Socket.IO Maximum call stack size exceeded - node.js

I've written a small Socket.IO server, which works fine, I can connect to it, I can send/receive messages, so everything is working ok. Just the relevant part of the code is presented here:
var RedisStore = require('socket.io/lib/stores/redis');
const pub = redis.createClient('127.0.0.1', 6379);
const sub = redis.createClient('127.0.0.1', 6379);
const store = redis.createClient('127.0.0.1', 6379);
io.configure(function() {
io.set('store', new RedisStore({
redisPub : pub,
redisSub : sub,
redisClient : store
}));
});
io.sockets.on('connection', function(socket) {
socket.on('message', function(msg) {
pub.publish("lobby", msg);
});
/*
* Subscribe to the lobby and receive messages.
*/
var sub = redis.createClient('127.0.0.1', 6379);
sub.subscribe("lobby");
sub.on('message', function(channel, msg) {
socket.send(msg);
});
});
I've also written a script presented below that connects to the server and spawns connections in the setInterval function, which spawns a new connection each 10milisecons, so it's spawning quite a lot of connections.
#!/usr/bin/env node
var io = require('socket.io-client');
var reconn = {'force new connection': true};
var sockets = [];
var num = 1000;
function startSocket(i) {
sockets[i] = io.connect("http://127.0.0.1:8080", reconn);
sockets[i].on('connect', function() {
console.log("Socket["+i+"] connected.");
});
sockets[i].on('message', function(msg) {
console.log("Socket["+i+"] Message received: "+msg);
});
}
/*
* Start number of sockets.
*/
for(var i=0; i<num; i++) {
startSocket(i);
}
/*
* Send messages forever.
*/
setInterval(function() {
for(var i=0; i<num; i++) {
sockets[i].send("Hello from socket "+i+".");
}
}, 10);
This script is a benchmark tool spawning 1000 connections to the server, but when running for several minutes, the server dies with the following error message:
node.js:0 // Copyright Joyent, Inc. and other Node contributors. ^
RangeError: Maximum call stack size exceeded
I know that there's not enough stack space available so the exception occurs and the process is terminated, but even if I enlarge the stack with the --stack-size variable, this doesn't actually solve the problem, because I can always spawn more connections, which will eventually kill the server.
My question is: how can I prevent this. This is an effective DoS scenario, where anybody can hack together this little script and force the node server to terminate, but I would like to prevent this from happening. I would like Node server to never terminate, just process messages slowly.
Any ideas if this can be prevented. I'm not sure that I would like to block IPs, since I would also like mobile phones to login to the system, where many of them use the same IP, so the node server can mistakenly think a DoS is being in place by one mobile network operation and blocks its IP.
Thank you

If you would like your node server to run forever, no matter what, use https://github.com/nodejitsu/forever
As for the Exception - My hunch is that var sub = redis.createClient('127.0.0.1', 6379); may allocate a variable in the stack each time a connection is established.
I would first try to put var subs = [] in the global scope and
subs[socket.id] = redis.createClient('127.0.0.1', 6379);
Or something like socket.sub = redis.createClient('127.0.0.1', 6379); to piggyback the existing, hopefully heap based, socket.io data structures.
If not working, try to isolate the problem by removing the use of Redis...

Related

socket.on event gets triggered multiple times

var express = require('express');
var app = express();
var server = app.listen(3000);
var replyFromBot;
app.use(express.static('public'));
var socket = require('socket.io');
var io = socket(server);
io.sockets.on('connection' , newConnection);
function newConnection(socket) {
console.log(socket.id);
listen = true;
socket.on('Quest' ,reply);
function reply(data) {
replyFromBot = bot.reply("local-user", data);
console.log(socket.id+ " "+replyFromBot);
socket.emit('Ans' , replyFromBot);
}
}
i've created a server based chat-bot application using node.js socket.io and express but the thing is for first time when i call socket.on it gets executed once and for 2nd time it gets executed twice for 3rd thrice and so on i've tackled this issue by setting a flag on my client so that it would display only once. i just wants to know is my code logically correct i mean is this a good code? because if the client ask a question for 10th time than listeners array will have 10+9+8....+1 listeners it would go on increasing depending upon number of questions clients asked. which is not good
i tried using removeListener it just removes listener once and it dosent call back for 2nd time. what do you guys recommend? do i go with this or is there any other way to add the listener when socket.on called and remove it when it gets executed and again add listener for the next time it gets called
thank-you.
client code:
function reply() {
socket.emit('Quest' , Quest);
flag = true;
audio.play();
socket.on('Ans', function(replyFromBot) {
if(flag) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
flag = false;
}
});
}
The problem is caused by your client code. Each time you call the reply() function in the client you set up an additional socket.on('Ans', ...) event handler which means they accumulate. You can change that to socket.once() and it will remove itself each time after it get the Ans message. You can then also remove your flag variable.
function reply() {
socket.emit('Quest' , Quest);
audio.play();
// change this to .once()
socket.once('Ans', function(replyFromBot) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
});
}
Socket.io is not really built as a request/response system which is what you are trying to use it as. An even better way to implement this would be to use the ack capability that socket.io has so you can get a direct response back to your Quest message you send.
You also need to fix your shared variables replyFromBot and listen on your server because those are concurrency problems waiting to happen as soon as you have multiple users using your server.
Better Solution
A better solution would be to use the ack capability that socket.io has to get a direct response to a message you sent. To do that, you'd change your server to this:
function newConnection(socket) {
console.log(socket.id);
socket.on('Quest', function(data, fn) {
let replyFromBot = bot.reply("local-user", data);
console.log(socket.id+ " "+replyFromBot);
// send ack response
fn(replyFromBot);
});
}
And, change your client code to this:
function reply() {
audio.play();
socket.emit('Quest', Quest, function(replyFromBot) {
console.log("hi");
var para = document.createElement("p2");
x = document.getElementById("MiddleBox");
para.appendChild(document.createTextNode(replyFromBot));
x.appendChild(para);
x.scrollTop = x.scrollHeight;
});
}
Doing it this way, you're hooking into a direct reply from the message so it works as request/response much better than the way you were doing it.
Instead of socket.on('Quest' ,reply); try socket.once('Quest' ,reply);
The bug in your code is that each time newConnection() is called node registers a event listener 'Quest'. So first time newConnection() is called the number of event listener with event 'Quest' is one, the second time function is called, number of event listener increases to two and so on
socket.once() ensures that number of event listener bound to socket with event 'Quest' registered is exactly one

How to use redis with node js clusters

I have been using redis as an in-memory store with my nodejs server.
Sessions are also being managed with redis.
Currently what i have been doing is, i flush my redis whenever my server connects to it, so that no session is there whenever server starts
Like this:
redisClient.on('connect', function () {
redisClient.flushdb(function (err, succeeded) {
logger.debug("redis db cleared on startup--", succeeded); // will be true if successfull
});
});
I also use redis for some other data storage like some queue.
But now i want to implement clustering on my server.
My problem is if i have 4 cores, 4 instances of node will be runnig on my server.
num_processes = require('os').cpus().length;
if (cluster.isMaster) {
console.log(`Master ${process.pid} is running`);
var workers = [];
// Helper function for spawning worker at index 'i'.
var spawn = function(i) {
console.log("spawing at index---",i);
workers[i] = cluster.fork();
console.log("----worker id-------", workers[i].id,"-------");
// Optional: Restart worker on exit
workers[i].on('exit', function(code, signal) {
console.log(`code is ${code} and signal is ${signal}`)
console.log(`worker ${workers[i].process.pid} died array index is ---${i}`);
console.log('respawning worker', i);
spawn(i);
});
workers[i].on('listening', () => {
workers[i].send({'index' : i });
});
};
// Spawn workers.
for (var i = 0; i < num_processes; i++) {
spawn(i);
} // Code to run if we're in a worker process
} else {
var redis = require('redis');
const sio_redis = require('socket.io-redis'),
redisClient = redis.createClient();
redisClient.on('connect', function () {
redisClient.flushdb(function (err, succeeded) {
logger.debug("redis db cleared on startup--", succeeded); // will be true if successfull
});
});
var RedisStore = require('connect-redis')(session);
const redisStore = new RedisStore({'host': 'localhost', 'port': 6379, 'client': redisClient});
var server = require('http').Server(app);
var listeningServer = server.listen(3002);
}
If any instance exit or die due to some reason , it will clear all the session and data in redis
I dont want this to happen. How should i work with redis in this scenario, so that sessions and data corresponding to that instance get cleared?
You can check if current process is master. That way it will only flush the first time you start your app. If any fork restarts then it wont flush the db.
redisClient = redis.createClient();
redisClient.on('connect', function() {
if (cluster.isMaster) {
redisClient.flushdb(function(err, succeeded) {
logger.debug("redis db cleared on startup--", succeeded); // will be true if successfull
});
}else{
logger.debug("Forked instance no need to flush redis db"); //
}
});
There are essentially two ways around this that I think are sane:
don't flush redis,
separate flush from instance startup.
don't flush
You say you want to cleanup session data on startup. Well, one of the points of having sessions not in memory is to persist them accross the sessions, and keep your actual app server (Node.js app) stateless.
You can, e.g. set expire key on all session data. So every time you "save" a session, you also setex on that session, optionally even prolonging this TTL every time you access the session (so, a session is valid 12 hours from last touching it, or something). Or, depending on your usage, your session middleware could do that, e.g. https://stackoverflow.com/a/45019589/162070.
Or not expire it at all.
flush independantly
Maybe you only flush sessions because you expect to have breaking changes and your old sessions won't work. Or have an explicit requirement, that on each new deploy, you clean the session data up. In any case, you separate this then. Have, for example, your server.js, as your app, and have a separate session-cleanup.js file that connects to redis, flushes, and disconnects. Then have npm setup like this:
"scripts": {
"session.cleanup": "node lib/session-cleanup.js",
"start": "npm run session.cleanup && node lib/server.js",
...
}
That way, before running your server.js and it runs cluster mode, you will cleanup sessions first. And if your cluster instances die and respawn, nothing happens.
Now, each time you redeploy, you clean it up. Or even remove this from "start", and have your deployment pipeline call npm run session.cleanup explicitly. You can even expose a button in an admin UI to do this on runtime (which would likely log you out :)), the point is, session cleanup is now a separate issue from server start.

Node Cluster issue using Socket.io and Redis

Ok, I have an express-powered API where I also have socket.io running to receive/send realtime events...all works just dandy. I need to cluster my app. I set everything up based on the below code. I spin up workers, they get connections and everything works, except the fact that now I can't "blast" to all socket.io connections. Here is the setup (taken from this):
var express = require('express'),
cluster = require('cluster'),
net = require('net'),
sio = require('socket.io'),
sio_redis = require('socket.io-redis');
var port = 3000,
num_processes = require('os').cpus().length;
if (cluster.isMaster) {
// This stores our workers. We need to keep them to be able to reference
// them based on source IP address. It's also useful for auto-restart,
// for example.
var workers = [];
// Helper function for spawning worker at index 'i'.
var spawn = function(i) {
workers[i] = cluster.fork();
// Optional: Restart worker on exit
workers[i].on('exit', function(worker, code, signal) {
console.log('respawning worker', i);
spawn(i);
});
};
// Spawn workers.
for (var i = 0; i < num_processes; i++) {
spawn(i);
}
// Helper function for getting a worker index based on IP address.
// This is a hot path so it should be really fast. The way it works
// is by converting the IP address to a number by removing the dots,
// then compressing it to the number of slots we have.
//
// Compared against "real" hashing (from the sticky-session code) and
// "real" IP number conversion, this function is on par in terms of
// worker index distribution only much faster.
var workerIndex = function (ip, len) {
var _ip = ip.split(/['.'|':']/),
arr = [];
for (el in _ip) {
if (_ip[el] == '') {
arr.push(0);
}
else {
arr.push(parseInt(_ip[el], 16));
}
}
return Number(arr.join('')) % len;
}
// Create the outside facing server listening on our port.
var server = net.createServer({ pauseOnConnect: true }, function(connection) {
// We received a connection and need to pass it to the appropriate
// worker. Get the worker for this connection's source IP and pass
// it the connection.
var worker = workers[worker_index(connection.remoteAddress, num_processes)];
worker.send('sticky-session:connection', connection);
}).listen(port);
} else {
// Note we don't use a port here because the master listens on it for us.
var app = new express();
// Here you might use middleware, attach routes, etc.
// Don't expose our internal server to the outside.
var server = app.listen(0, 'localhost'),
io = sio(server);
// Tell Socket.IO to use the redis adapter. By default, the redis
// server is assumed to be on localhost:6379. You don't have to
// specify them explicitly unless you want to change them.
io.adapter(sio_redis({ host: 'localhost', port: 6379 }));
// Here you might use Socket.IO middleware for authorization etc.
// Listen to messages sent from the master. Ignore everything else.
process.on('message', function(message, connection) {
if (message !== 'sticky-session:connection') {
return;
}
// Emulate a connection event on the server by emitting the
// event with the connection the master sent us.
server.emit('connection', connection);
connection.resume();
});
}
So I connect from various machines to test concurrency, workers do their thing and all is good, but when I get an IO connection, I'm logging the TOTAL "connected" count and it's always 1 per instance. I need a way to say
allClusterForks.emit(stuff)
I get the connection on the correct worker pid, but "ALL CONNECTIONS" always returns 1.
io.on('connection', function(socket) {
console.log('Connected to worker %s', process.pid);
console.log("Adapter ROOMS %s ", io.sockets.adapter.rooms);
console.log("Adapter SIDS %s ", io.sockets.adapter.sids);
console.log("SOCKETS CONNECTED %s ", Object.keys(io.sockets.connected).length);
});
I can see the subscribe/unsubscribe coming in using Redis MONITOR
1454701383.188231 [0 127.0.0.1:63150] "subscribe" "socket.io#/#gXJscUUuVQGzsYJfAAAA#"
1454701419.130100 [0 127.0.0.1:63167] "subscribe" "socket.io#/#geYSvYSd5zASi7egAAAA#"
1454701433.842727 [0 127.0.0.1:63167] "unsubscribe" "socket.io#/#geYSvYSd5zASi7egAAAA#"
1454701444.630427 [0 127.0.0.1:63150] "unsubscribe" "socket.io#/#gXJscUUuVQGzsYJfAAAA#"
These are connections from 2 different machines, I would expect by using the socket io redis adapter that these subscriptions would be coming in on the same redis connection, but they are different.
Am I just totally missing something? There's a surprising lack of documentation/articles out there for this that aren't either completely outdated/wrong/ambiguous.
EDIT:
Node v5.3.0
Redis v3.0.6
Socket.io v1.3.7
So if anyone comes across this, I figured out that actually "looking" at the counts of connected sockets across processes is not a thing, but broadcasting or emitting to them is. So I've basically just been "testing" for no reason. All works as expected. I WILL be rewriting the socket.io-redis adapter to allow checking counts across processes.
There was a pull request a few years ago to implement support for what I was trying to do. https://github.com/socketio/socket.io-redis/pull/15 and I might try cleaning that up and re-submitting.

Node.js cluster module appears to break Socket.io handshake

I have the following simple WebSocket server built around the Socket.io library:
var PROCESSES = 1,
cluster = require('cluster'),
i;
if (cluster.isMaster) {
for (i = 0; i < PROCESSES; i++) {
console.log('Forking worker', i);
cluster.fork();
}
} else {
(function () {
var server = require('http').Server(),
io = require('socket.io')(server);
io.on('connection', function (socket) {
socket.on('message', function (message) {
socket.emit('message', message + ' too!');
});
});
server.listen(8080);
})();
}
When started, it creates a single server process which listens for WebSocket connections and echoes a variation of the message back to the client:
$ iocat --socketio ws://localhost:8080
> i am hungry
i am hungry too!
> i like you
i like you too!
>
Now, when I change the PROCESSES variable to a number larger than 1, the client can no longer connect.
var PROCESSES = 2,
...
...results in...
$ iocat --socketio ws://localhost:8080
> client.on error
$ iocat -v --socketio ws://localhost:8080
> SIOClient> SIOClient: url-> ws://localhost:8080
SIOClient> onError { [Error: xhr poll error] description: 400 }
client.on error
My gut feeling is that the cluster module, when given more than one worker process, inappropriately switches from one process to another mid-handshake. But I would have thought that the entire connection, from the client initiating the handshake to the closing of the socket at the very very end, occurred over one persistent, keep-alive'd connection.
So what exactly is going on here? And how could it be worked around? I'm familiar with the idea of using a Redis store to share state between server processes on different machines, but that feels like too much infrastructure for my use case (collecting a stream of events from the client and replying with an acknowledgement).
Versions: socket.io#1.3.3, node#0.10.36, seen on OS X 10.10 and CentOS 6.6
socket.io is not a simple wrapper over WebSockets, it does much more. The opening handshake is an http request to decide on a protocol (WebSocket, polling, flash sockets, etc.) followed by, in your case, probably a WebSocket request. If those hit different processes, the handshake will fail.
socket.io requires that you use sticky sessions, to ensure that a given client hits the same process each time. They suggest using the sticky-session module if you want to use cluster.

NodeJS on multiple processors (PM2, Cluster, Recluster, Naught)

I am investigating options for running node in a multi-core environment.
I'm trying to determine the best approach and so far I've seen these options
Use built in cluster library to spin up works and respond to signals
Use PM but, PM2 -i is listed as beta.
Naught
Recluster
Are there other alternatives? What are folks using in production?
I've been using the default cluster library, and it works very well. I've had over 10,000 concurrents(multiple clusters on multiple servers) and it works very well.
It is suggested to use clusters with domain for error handling.
This is lifted straight from http://nodejs.org/api/domain.html I've mades some changes on how it spawns new clusters for each core of your machine. and got rid of if/else and added express.
var cluster = require('cluster'),
http = require('http'),
PORT = process.env.PORT || 1337,
os = require('os'),
server;
function forkClusters () {
var cpuCount = os.cpus().length;
// Create a worker for each CPU
for (var i = 0; i < cpuCount ; i += 1) {
cluster.fork();
}
}
// Master Process
if (cluster.isMaster) {
// You can also of course get a bit fancier about logging, and
// implement whatever custom logic you need to prevent DoS
// attacks and other bad behavior.
//
// See the options in the cluster documentation.
//
// The important thing is that the master does very little,
// increasing our resilience to unexpected errors.
forkClusters ()
cluster.on('disconnect', function(worker) {
console.error('disconnect!');
cluster.fork();
});
}
function handleError (d) {
d.on('error', function(er) {
console.error('error', er.stack);
// Note: we're in dangerous territory!
// By definition, something unexpected occurred,
// which we probably didn't want.
// Anything can happen now!Be very careful!
try {
// make sure we close down within 30 seconds
var killtimer = setTimeout(function() {
process.exit(1);
}, 30000);
// But don't keep the process open just for that!
killtimer.unref();
// stop taking new requests.
server.close();
// Let the master know we're dead.This will trigger a
// 'disconnect' in the cluster master, and then it will fork
// a new worker.
cluster.worker.disconnect();
} catch (er2) {
// oh well, not much we can do at this point.
console.error('Error sending 500!', er2.stack);
}
});
}
// child Process
if (cluster.isWorker) {
// the worker
//
// This is where we put our bugs!
var domain = require('domain');
var express = require('express');
var app = express();
app.set('port', PORT);
// See the cluster documentation for more details about using
// worker processes to serve requests.How it works, caveats, etc.
var d = domain.create();
handleError(d);
// Now run the handler function in the domain.
//
// put all code here. any code included outside of domain.run will not handle errors on the domain level, but will crash the app.
//
d.run(function() {
// this is where we start our server
server = http.createServer(app).listen(app.get('port'), function () {
console.log('Cluster %s listening on port %s', cluster.worker.id, app.get('port'));
});
});
}
We use Supervisor to manage our Node.JS process's, to start them upon boot, and to act as a watchdog in case the process's crash.
We use Nginx as a reverse-proxy to load balance traffic between the process's that listen to different ports
this way each process is isolated from the other.
for example: Nginx listens on port 80 and forwards traffic to ports 8000-8003
I was using PM2 for quite a while, but their pricing is expensive for my needs as I'm having my own analytics environment and I don't require support, so I decided to experiment alternatives. For my case, just forever made the trick, very simple one actually:
forever -m 5 app.js
Another useful example is
forever start app.js -p 8080

Resources