Socket IO infinite loop over 1000 connections - node.js

I need to benchmark multiples socket connections. I've got a nodejs server with this following code :
var io = require('./lib/node_modules/socket.io').listen(12345)
io.sockets.on("connect", function(socket) {
console.log("Socket " + socket.id + " connected.")
socket.on("disconnect", function() {
console.log("Socket " + socket.id +" disconnected.")
})
})
and a nodejs client :
var port = 12345
, nbSocket = 1000
, io = require("./lib/node_modules/socket.io-client")
for (var i = 1;i <= nbSocket;i++)
{
var socket = io.connect("http://<<my_ip>>:" + port, {forceNew: true})
}
When client code was executed, server correctly connects sockets and ends normally.
But if we change nbSocket to 2000, server never ends connecting and disconnecting sockets.
We already tried to change the limit with :
ulimit -n5000
But it didn't worked. Is there another limit somewhere or something we missed ?

I tested on OSX running Node v0.12.4 and socket.io v1.3.5 and it started to cause me problems around nbSocket=5000.
Try appending this snippet to the end of your server script:
process.on('uncaughtException', function(err) {
console.info(util.inspect(err, {colors: true}));
});
Also, I changed your code a little and added a timer that twice every second prints the number of open sockets:
var
util = require('util'),
io = require('socket.io').listen(12345);
var
clientCount = 0;
function onClientDisconnect() {
clientCount--;
}
io.on('connect', function(socket) {
clientCount++;
socket.on('disconnect', onClientDisconnect);
});
console.info('Listening...');
setInterval(function () {
console.info('Number of open sockets: %d', clientCount);
}, 500);
process.on('uncaughtException', function(err) {
console.info(util.inspect(err, {colors: true}));
});
When the number of open sockets started to get close to 5000, I started seeing these 2 messages several times:
{ [Error: accept ENFILE] code: 'ENFILE', errno: 'ENFILE', syscall: 'accept' }
{ [Error: accept EMFILE] code: 'EMFILE', errno: 'EMFILE', syscall: 'accept' }
According to libc manual:
ENFILE: too many distinct file openings in the entire system
EMFILE: the current process has too many files open
So in fact my problem was the limit of file descriptors, so check if it's also your problem by appending the above snippet to your server script. If the exceptions appear, you should investigate how to properly increase the limit of open files in your system.

Related

Error EADDRNOTAVAIL on socket.send() in node (SSDP protocol)

I'm trying to implement a UPnP discovery service tool (SSDP protocol), I did something similar in python following this post: https://www.electricmonk.nl/log/2016/07/05/exploring-upnp-with-python/ and I would like to port it to node (v. 8.6.0) and typescript however I'm getting the following error when I try to send the message (socket.send(...)):
{ Error: send EADDRNOTAVAIL 239.255.255.250:1900
at Object._errnoException (util.js:1019:11)
at _exceptionWithHostPort (util.js:1041:20)
at SendWrap.afterSend [as oncomplete] (dgram.js:475:11)
code: 'EADDRNOTAVAIL',
errno: 'EADDRNOTAVAIL',
syscall: 'send',
address: '239.255.255.250',
port: 1900 }
I've found a snippet of code for node, that makes this exact thing (https://coolaj86.com/articles/adventures-in-upnp-with-node-js/) and I think that my code is quite equivalent however I cannot see why my code isn't working
const dgram = require('dgram');
const socket = dgram.createSocket('udp4');
let msg_txt = 'M-SEARCH * HTTP/1.1\r\n' +
'HOST:239.255.255.250:1900\r\n' +
'ST:upnp:rootdevice\r\n' +
'MX:2\r\n' +
'MAN:"ssdp:discover"\r\n\r\n';
const message = Buffer.from(msg_txt);
socket.on('message', (msg: Buffer, info: any) => {
console.log(msg.toString());
});
socket.bind({
address: '239.255.255.250',
port: 1900
}, (err) => {
!!err && console.error(err);
});
socket.on('listening', () => {
console.log('Sending msg...');
socket.send(message, 0, message.length, 1900, '239.255.255.250', (err) => {
!!err && console.error(err); // err != null
});
});
I suspect that is a typical one-line-problem but after a while I couldn't find it out, any help is welcome.
I saw it, in node the API is a bit different, the binding should be against the '0.0.0.0' iface and the port 0 (for a random number), so changing the binding command to the following code just fix it:
socket.bind({
address: '0.0.0.0',
port: 0
}, (err) => {
!!err && console.error(err);
});
In python I called to socket.recvfrom() method to get the UPnP devices responses, there is not an explicit socket binding.
For what it's worth, I had a similar setup using Node 10 on Windows and creating a udp4 socket on localhost. Weirdly enough, the setup described in the accepted answer worked on every OS and Node version except Node 10 on Windows in my CI tests. In order to get things to work, I had to explicitly bind to the address localhost instead of 0.0.0.0, like so:
const socket = dgram.createSocket('udp4').unref();
socket.bind({
address: 'localhost',
port: 0
}, (err) => {
!!err && console.error(err);
});

Nodejs HTTPS client timeout not closing TCP connection

I need to close http connection if they take longer than 3s, so this is my code:
var options = {
host: 'google.com',
port: '81',
path: ''
};
callback = function(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
response.on('error', function () {
console.log('ERROR!');
});
}
var req = https.request(options, callback);
req.on('socket', function(socket) {
socket.setTimeout(3000);
socket.on('timeout', function() {
console.log('Call timed out!');
req.abort();
//req.end();
//req.destroy();
});
});
req.on('error', function(err) {
console.log('REQUEST ERROR');
console.dir(err);
req.abort();
//req.end();
});
req.end();
This is what I get after 3s:
Call timed out!
REQUEST ERROR
{ [Error: socket hang up] code: 'ECONNRESET' }
Using a watch on lsof | grep TCP | wc -l I can see that the TCP connection remains open, even after receiving the 'timeout' event.
After an eternity, I get this and the connection is closed:
REQUEST ERROR
{ [Error: connect ETIMEDOUT] code: 'ETIMEDOUT', errno: 'ETIMEDOUT', syscall: 'connect' }
Does anyone know why this is happening? Why does calling req.abort() or req.end() or req.destory() not close the connection? Is it because I'm setting the timeout on the socket instead of the actual HTTP call? If yes, how do I close the connection?
you need to set the timeout on the connection:
req.connection.setTimeout(3000);
This timeout will change the socket status from ESTABLISHED to FIN_WAIT1 and FIN_WAIT2.
In Ubuntu there is a default timeout of 60 seconds for FIN_WAIT socket status, so the total time for the socket to close is 63 seconds if it doesn't receive any traffic. If the sockets receive traffic, the timeouts will start over.
If you need to close the socket within 3 seconds, I guess you have to set the connection timeout to 3000ms and lower the kernel tcp fin wait timeout.

Why NodeJS KeepAlive does not seem to work as expected?

Quoted from TCP keepalive HowTo:
In order to understand what TCP keepalive (which we will just call
keepalive) does, you need do nothing more than read the name: keep TCP
alive. This means that you will be able to check your connected socket
(also known as TCP sockets), and determine whether the connection is
still up and running or if it has broken.
So why is the following code not throwing something when the internet connection is broken?
var tls = require('tls');
var socket = tls.connect(443, "google.com", function connected() {
console.log('connected');
});
socket.setNoDelay(true);
socket.setKeepAlive(true, 0);
socket.setTimeout(0, function(){
console.log('timeout');
});
socket.on('data', function(data) {
console.log(data);
});
socket.on('close', function() {
console.error("close");
});
socket.on('error', function(err) {
console.error("error", err);
});
Tested on MacOS/Debian, with NodeJS v0.10.17
Quoting man 7 tcp:
tcp_keepalive_time (integer; default: 7200; since Linux 2.2)
The number of seconds a connection needs to be idle before TCP begins sending out keep-alive probes. Keep-alives are only sent when the SO_KEEPALIVE socket option is enabled. The default value is 7200 seconds (2 hours). An idle connection is terminated after approximately an additional 11 minutes (9 probes an interval of 75 seconds apart) when keep-alive is enabled.
So after ~10 minutes (on MacOS 10.8) node emitted an error:
error { [Error: read ETIMEDOUT] code: 'ETIMEDOUT', errno: 'ETIMEDOUT', syscall: 'read' }
https://www.npmjs.com/package/net-keepalive
Here is a module which lets you configure TCP_KEEPINTVL and TCP_KEEPCNT per-socket basis.
Provides high-level access to socket options like TCP_KEEPIDLE,
TCP_KEEPINTVL, TCP_KEEPCNT
var Net = require('net')
, NetKeepAlive = require('net-keepalive')
;
// Create a TCP Server
var srv = Net.createServer(function(s){>
console.log('Connected %j', s.address())
// Doesn't matter what it does
s.pipe(s)
});
// Start on some port
srv.listen(1337, function(){
console.log('Listening on %j', srv.address())
});
// Connect to that server
var s = Net.createConnection({port:1337}, function(){
console.log('Connected to %j', s.address())
//IMPORTANT: KeepAlive must be enabled for this to work
s.setKeepAlive(true, 1000)
// Set TCP_KEEPINTVL for this specific socket
NetKeepAlive.setKeepAliveInterval(s, 1000)
// and TCP_KEEPCNT
NetKeepAlive.setKeepAliveProbes(s, 1)
});

socket.io client persistent retries to unreachable host

I'm trying to get a persistent connection from my socket.io-client (running on Node.js) to a remote websocket. I do not have control over the remote socket, and sometimes it can go down entirely. I would like to attempt to reconnect() whenever an error or disconnect occurs. In the following example, I'm trying to test the case where the remote host is refusing a connection. In this case, I would like to attempt to reconnect after 1 second. It calls a second time, and exits.
Here's the code:
var events = require('events'),
util = require('util'),
io = require('socket.io-client'),
url = "ws://localhost:12345", // intentionally an unreachable URL
socketOptions = {
"transports" : [ "websocket" ],
"try multiple transports" : false,
"reconnect" : false,
"connect timeout" : 5000
};
// The goal is to have this socket attempt to connect forever
// I would like to do it without the built in reconnects, as these
// are somewhat unreliable (reconnect* events not always firing)
function Test(){
var self = this;
events.EventEmitter.call(self);
var socket;
function reconnect(){
setTimeout(go, 1000);
}
function go(){
console.log("connecting to", url, socketOptions);
socket = io.connect(url, socketOptions);
socket.on('connect', function(){
console.log("connected! wat.");
});
socket.on('error', function(err){
console.log("socket.io-client 'error'", err);
reconnect();
});
socket.on('connect_failed', function(){
console.log("socket.io-client 'connect_failed'");
reconnect();
});
socket.on('disconnect', function(){
console.log("socket.io-client 'disconnect'");
reconnect();
});
}
go();
}
util.inherits(Test, events.EventEmitter);
var test = new Test();
process.on('exit', function(){
console.log("this should never end");
});
When running it under node 0.11.0 I get the following:
$ node socketio_websocket.js
connecting to ws://localhost:12345 { transports: [ 'websocket' ],
'try multiple transports': false,
reconnect: false,
'connect timeout': 5000 }
socket.io-client 'error' Error: connect ECONNREFUSED
at errnoException (net.js:878:11)
at Object.afterConnect [as oncomplete] (net.js:869:19)
connecting to ws://localhost:12345 { transports: [ 'websocket' ],
'try multiple transports': false,
reconnect: false,
'connect timeout': 5000 }
this should never end
The ECONNREFUSED is an exception you don't manage.
Try with this:
process.on('uncaughtException', function(err) {
if(err.code == 'ECONNREFUSED'){
reconnect();
}
}
Edit
Modify the options like this:
socketOptions = {
"transports" : [ "websocket" ],
"try multiple transports" : false,
"reconnect" : false,
'force new connection': true, // <-- Add this!
"connect timeout" : 5000
};
and the reconnect function (look in the comments for the explanation)
function reconnect(){
socket.removeAllListeners();
setTimeout(go, 1000);
}
Probably socket.io reuse the same connection without creating a new one, forcing it the app works

How to deal with 'read ETIMEDOUT' in Node.js?

I have a pub/sub model using Node.js to transmit data from one client to another client. Besides, the server also records everything received and sends it to new clients.
However, some data corrupted when transfer, and I got error like:
Error with socket!
{ [Error: write EPIPE] code: 'EPIPE', errno: 'EPIPE', syscall: 'write' }
Error with socket!
{ [Error: read ETIMEDOUT] code: 'ETIMEDOUT', errno: 'ETIMEDOUT', syscall: 'read' }
I don't know how to properly handle these errors. It looks like the client is down.
Since the server is only a proxy like a server, it doesn't really know what data means. I have no idea how to validate every data pack before meeting these errors.
Here is my code:
// server is an object inheriting from net.Server
server.on('listening', function() {
var port = server.address().port;
}).on('connection', function(cli) {
cli.socketBuf = new Buffers();
cli.commandStarted = false;
cli.dataSize = 0;
cli.setKeepAlive(true, 10*1000);
cli.setNoDelay(true);
cli.on('connect', function() {
server.clients.push(cli);
}).on('close', function() {
var index = server.clients.indexOf(cli);
server.clients.splice(index, 1);
}).on('data', function (buf) {
server.emit('data', cli, buf);
if(op.autoBroadcast) {
_.each(server.clients, function(c) {
if(c != cli) c.write(buf);
});
}
}).on('error', function(err) {
console.log('Error with socket!');
console.log(err);
});
}).on('error', function(err) {
console.log('Error with server!');
console.log(err);
});
// ...
// room.dataSocket is an instance of server beyond
room.dataSocket.on('data', function(cli, d) {
// bf is a buffered file
bf.append(d);
room.dataFileSize += d.length;
}).on('connection', function(con){
bf.readAll(function(da) {
con.write(da);
});
});
If you get an EPIPE or indeed any error when writing, the peer has closed or the connection has been dropped. So you must close the connection at that point.
If you get a read timeout the inference is that either you have set an unrealistically short timeout or else the peer has failed to deliver in time: in the second case once again you should assume the connection is down, and close it.

Resources