Why NodeJS KeepAlive does not seem to work as expected? - node.js

Quoted from TCP keepalive HowTo:
In order to understand what TCP keepalive (which we will just call
keepalive) does, you need do nothing more than read the name: keep TCP
alive. This means that you will be able to check your connected socket
(also known as TCP sockets), and determine whether the connection is
still up and running or if it has broken.
So why is the following code not throwing something when the internet connection is broken?
var tls = require('tls');
var socket = tls.connect(443, "google.com", function connected() {
console.log('connected');
});
socket.setNoDelay(true);
socket.setKeepAlive(true, 0);
socket.setTimeout(0, function(){
console.log('timeout');
});
socket.on('data', function(data) {
console.log(data);
});
socket.on('close', function() {
console.error("close");
});
socket.on('error', function(err) {
console.error("error", err);
});
Tested on MacOS/Debian, with NodeJS v0.10.17

Quoting man 7 tcp:
tcp_keepalive_time (integer; default: 7200; since Linux 2.2)
The number of seconds a connection needs to be idle before TCP begins sending out keep-alive probes. Keep-alives are only sent when the SO_KEEPALIVE socket option is enabled. The default value is 7200 seconds (2 hours). An idle connection is terminated after approximately an additional 11 minutes (9 probes an interval of 75 seconds apart) when keep-alive is enabled.
So after ~10 minutes (on MacOS 10.8) node emitted an error:
error { [Error: read ETIMEDOUT] code: 'ETIMEDOUT', errno: 'ETIMEDOUT', syscall: 'read' }

https://www.npmjs.com/package/net-keepalive
Here is a module which lets you configure TCP_KEEPINTVL and TCP_KEEPCNT per-socket basis.
Provides high-level access to socket options like TCP_KEEPIDLE,
TCP_KEEPINTVL, TCP_KEEPCNT
var Net = require('net')
, NetKeepAlive = require('net-keepalive')
;
// Create a TCP Server
var srv = Net.createServer(function(s){>
console.log('Connected %j', s.address())
// Doesn't matter what it does
s.pipe(s)
});
// Start on some port
srv.listen(1337, function(){
console.log('Listening on %j', srv.address())
});
// Connect to that server
var s = Net.createConnection({port:1337}, function(){
console.log('Connected to %j', s.address())
//IMPORTANT: KeepAlive must be enabled for this to work
s.setKeepAlive(true, 1000)
// Set TCP_KEEPINTVL for this specific socket
NetKeepAlive.setKeepAliveInterval(s, 1000)
// and TCP_KEEPCNT
NetKeepAlive.setKeepAliveProbes(s, 1)
});

Related

Setting Heartbeat when connecting to RabbitMQ via Amqplib

When connecting to RabbitMQ via amqplib, the value of the Heartbeat parameter cannot be set.
I do this:
// NPM Modules
const amqp = require('amqplib/callback_api');
const withAutoRecovery = require('amqplib-auto-recovery');
// Application configuration
const config = settings.getConfig();
withAutoRecovery(amqp, {
onError: (err) => { console.log(err.message); },
isErrorUnrecoverable: (err) => { console.log(err) }
}).connect(config.rabbitmq, (err, connection) => {
// If there are connection errors
if (err) {
console.error(this.conn_str_amqp);
console.error('[AMQP] Connection error:', err.message);
} else if (connection) {
// Connection Error Event
connection.on('error', (err) => {
if (err.message !== 'Connection closing') {
console.error('[AMQP] Closing connection', err.message);
}
});
// Error event when you close the connection
connection.on('close', () => {
console.error('[AMQP] Closing connection');
});
// Hint to user
console.log('[AMQP] Connected');
}
});
Here config.rabbitmq is:
{
protocol: 'amqp',
hostname: 'localhost',
port: 5672,
username: 'guest',
password: 'guest',
locale: 'en_US',
frameMax: 0x1000,
heartbeat: 1800,
vhost: '/',
}
I expect to get the Heartbeat = 1800s parameter value, in the UI Managment window the default value Heartbeat = 60s is displayed.
Screenshot:
I tried to pass another object instead config.rabbitmq line (https://www.rabbitmq.com/uri-spec.html and https://www.rabbitmq.com/uri-query-parameters.html) in the format:
function getRabbitMQConnectionString() {
const rabbitmq = config.rabbitmq;
return `${rabbitmq.protocol}://${rabbitmq.username}:${rabbitmq.password}#${rabbitmq.hostname}:${rabbitmq.port}${encodeURIComponent(rabbitmq.vhost)}?heartbeat=${rabbitmq.heartbeat}`;
}
Thus, too, it turned out similar to the above effect.
Tell me, please, how to correctly set the Heartbeat parameter through the Node.js client application on amqplib?
For some reason I set up the heartbeat to 30 s and it worked.
From RabbitMQ documentation, they suggest not to set the heartbeats high, because it can disable them.
However, I don't know what is shown as 60s heartbeats.
https://www.rabbitmq.com/heartbeats.html#disabling
Reason
The reason of getting heartbeat 60s in Management UI is because the RabbitMQ server have default heartbeat 60s.
Explanation
And during connection creation the minimum of both client(amqplib) and server(RabbitMQ) heartbeat value is taken.
Solution
So, to increate the heartbeat value from amqplib, we need to have more or equal value set in RabbitMQ configurations
Methods to Set heartbeat value for RabbitMQ Server
How to change RabbitMQ Heartbeat without restart
https://www.rabbitmq.com/configure.html#:~:text=Max%20value%3A%20536870912-,heartbeat,-Value%20representing%20the
Reference for more detail
https://www.rabbitmq.com/heartbeats.html
https://www.rabbitmq.com/configure.html

Node 4.1.0 TLSSocket issues

Strange behavior in Node with TLSSocket and tls.connect.
var port = 7000;
var host = '94.125.182.252'; //freenode
var tls = require('tls');
var net = require('net');
var socket = new net.Socket();
var secure;
secure = new tls.TLSSocket( socket, {
isServer: false,
rejectUnauthorized: false
});
// edit (left out of original post, but present in my test code, whoops)
secure.connect( {
port: port,
host: host
});
secure.setEncoding( 'utf8' );
secure.on( 'connect' , function() {
console.log( 'connected' );
})
.on( 'secureConnect', function() {
console.log( 'secure connect' );
})
.on( 'error', function( e ) {
console.log( 'error', e );
})
.on( 'data', function( data ) {
console.log( data );
});
if ( secure.isPaused() ) {
console.log( 'socket was paused' );
secure.resume();
}
This doesn't even attempt to connect and no error messages are produced. I have wireshark monitoring and there is no activity captured.
A different approach:
secure = tls.connect( {
rejectUnauthorized: false,
host: host,
port: port,
socket: socket
});
Same story, nothing captured, no errors. If I remove the socket: socket aspect above it will connect. This makes some sense as the docs state that if the socket option is specified it will ignore port and host. The above works on my previous Node version( 0.12.7).
If I want to use the existing socket I have to tell it to connect before calling tls.connect.
socket.connect( {
port: port,
host: host
});
secure = tls.connect( {
rejectUnauthorized: false,
socket: socket
});
This doesn't seem proper.
Passing a connecting socket to tls.TLSSocket( socket, ...) seems to have no effect.
The 'connect' event is fired but I imagine that is not related to TLSSocket.
I could not get tls.TLSSocket(...) to work on previous Node iterations.
Stepping through with node debug did not expose any obvious problems.
The options for net.Socket([options]) don't seem to accept a port or host for configuring until you try to connect, and trying to connect before passing to tls.connect seems counter intuitive. It would suggest that is not the intended usage.
So my questions would be:
What am I doing wrong with tls.TLSSocket() or perhaps is it a bug?
Am I correct to assume that passing an existing socket into tls.connect() is for already established connections switching protocol? If not, whats the proper way to assign a port and host?
Edit:
As per suggestion:
secure = tls.connect( {
rejectUnauthorized: false,
socket: socket
});
socket.connect( {
port: port,
host: host
});
This works.
secure = new tls.TLSSocket( socket , {
isServer: false,
rejectUnauthorized: false
});
socket.connect( {
port: port,
host: host
});
Unfortunately this does not work. A 'connect' event is emitted, never a 'secureConnect' and never any other events or data.
In your first two (non-working) examples, you only created a socket and never started actually connected it. Add a socket.connect(); at the end of your original code and it should work fine.
tls.connect() when passed a plain socket, does not actually call socket.connect(); internally, it merely sets up to start listening for data on the socket so it can decrypt incoming data properly.

Socket IO infinite loop over 1000 connections

I need to benchmark multiples socket connections. I've got a nodejs server with this following code :
var io = require('./lib/node_modules/socket.io').listen(12345)
io.sockets.on("connect", function(socket) {
console.log("Socket " + socket.id + " connected.")
socket.on("disconnect", function() {
console.log("Socket " + socket.id +" disconnected.")
})
})
and a nodejs client :
var port = 12345
, nbSocket = 1000
, io = require("./lib/node_modules/socket.io-client")
for (var i = 1;i <= nbSocket;i++)
{
var socket = io.connect("http://<<my_ip>>:" + port, {forceNew: true})
}
When client code was executed, server correctly connects sockets and ends normally.
But if we change nbSocket to 2000, server never ends connecting and disconnecting sockets.
We already tried to change the limit with :
ulimit -n5000
But it didn't worked. Is there another limit somewhere or something we missed ?
I tested on OSX running Node v0.12.4 and socket.io v1.3.5 and it started to cause me problems around nbSocket=5000.
Try appending this snippet to the end of your server script:
process.on('uncaughtException', function(err) {
console.info(util.inspect(err, {colors: true}));
});
Also, I changed your code a little and added a timer that twice every second prints the number of open sockets:
var
util = require('util'),
io = require('socket.io').listen(12345);
var
clientCount = 0;
function onClientDisconnect() {
clientCount--;
}
io.on('connect', function(socket) {
clientCount++;
socket.on('disconnect', onClientDisconnect);
});
console.info('Listening...');
setInterval(function () {
console.info('Number of open sockets: %d', clientCount);
}, 500);
process.on('uncaughtException', function(err) {
console.info(util.inspect(err, {colors: true}));
});
When the number of open sockets started to get close to 5000, I started seeing these 2 messages several times:
{ [Error: accept ENFILE] code: 'ENFILE', errno: 'ENFILE', syscall: 'accept' }
{ [Error: accept EMFILE] code: 'EMFILE', errno: 'EMFILE', syscall: 'accept' }
According to libc manual:
ENFILE: too many distinct file openings in the entire system
EMFILE: the current process has too many files open
So in fact my problem was the limit of file descriptors, so check if it's also your problem by appending the above snippet to your server script. If the exceptions appear, you should investigate how to properly increase the limit of open files in your system.

Nodejs HTTPS client timeout not closing TCP connection

I need to close http connection if they take longer than 3s, so this is my code:
var options = {
host: 'google.com',
port: '81',
path: ''
};
callback = function(response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
});
response.on('error', function () {
console.log('ERROR!');
});
}
var req = https.request(options, callback);
req.on('socket', function(socket) {
socket.setTimeout(3000);
socket.on('timeout', function() {
console.log('Call timed out!');
req.abort();
//req.end();
//req.destroy();
});
});
req.on('error', function(err) {
console.log('REQUEST ERROR');
console.dir(err);
req.abort();
//req.end();
});
req.end();
This is what I get after 3s:
Call timed out!
REQUEST ERROR
{ [Error: socket hang up] code: 'ECONNRESET' }
Using a watch on lsof | grep TCP | wc -l I can see that the TCP connection remains open, even after receiving the 'timeout' event.
After an eternity, I get this and the connection is closed:
REQUEST ERROR
{ [Error: connect ETIMEDOUT] code: 'ETIMEDOUT', errno: 'ETIMEDOUT', syscall: 'connect' }
Does anyone know why this is happening? Why does calling req.abort() or req.end() or req.destory() not close the connection? Is it because I'm setting the timeout on the socket instead of the actual HTTP call? If yes, how do I close the connection?
you need to set the timeout on the connection:
req.connection.setTimeout(3000);
This timeout will change the socket status from ESTABLISHED to FIN_WAIT1 and FIN_WAIT2.
In Ubuntu there is a default timeout of 60 seconds for FIN_WAIT socket status, so the total time for the socket to close is 63 seconds if it doesn't receive any traffic. If the sockets receive traffic, the timeouts will start over.
If you need to close the socket within 3 seconds, I guess you have to set the connection timeout to 3000ms and lower the kernel tcp fin wait timeout.

socket.io client persistent retries to unreachable host

I'm trying to get a persistent connection from my socket.io-client (running on Node.js) to a remote websocket. I do not have control over the remote socket, and sometimes it can go down entirely. I would like to attempt to reconnect() whenever an error or disconnect occurs. In the following example, I'm trying to test the case where the remote host is refusing a connection. In this case, I would like to attempt to reconnect after 1 second. It calls a second time, and exits.
Here's the code:
var events = require('events'),
util = require('util'),
io = require('socket.io-client'),
url = "ws://localhost:12345", // intentionally an unreachable URL
socketOptions = {
"transports" : [ "websocket" ],
"try multiple transports" : false,
"reconnect" : false,
"connect timeout" : 5000
};
// The goal is to have this socket attempt to connect forever
// I would like to do it without the built in reconnects, as these
// are somewhat unreliable (reconnect* events not always firing)
function Test(){
var self = this;
events.EventEmitter.call(self);
var socket;
function reconnect(){
setTimeout(go, 1000);
}
function go(){
console.log("connecting to", url, socketOptions);
socket = io.connect(url, socketOptions);
socket.on('connect', function(){
console.log("connected! wat.");
});
socket.on('error', function(err){
console.log("socket.io-client 'error'", err);
reconnect();
});
socket.on('connect_failed', function(){
console.log("socket.io-client 'connect_failed'");
reconnect();
});
socket.on('disconnect', function(){
console.log("socket.io-client 'disconnect'");
reconnect();
});
}
go();
}
util.inherits(Test, events.EventEmitter);
var test = new Test();
process.on('exit', function(){
console.log("this should never end");
});
When running it under node 0.11.0 I get the following:
$ node socketio_websocket.js
connecting to ws://localhost:12345 { transports: [ 'websocket' ],
'try multiple transports': false,
reconnect: false,
'connect timeout': 5000 }
socket.io-client 'error' Error: connect ECONNREFUSED
at errnoException (net.js:878:11)
at Object.afterConnect [as oncomplete] (net.js:869:19)
connecting to ws://localhost:12345 { transports: [ 'websocket' ],
'try multiple transports': false,
reconnect: false,
'connect timeout': 5000 }
this should never end
The ECONNREFUSED is an exception you don't manage.
Try with this:
process.on('uncaughtException', function(err) {
if(err.code == 'ECONNREFUSED'){
reconnect();
}
}
Edit
Modify the options like this:
socketOptions = {
"transports" : [ "websocket" ],
"try multiple transports" : false,
"reconnect" : false,
'force new connection': true, // <-- Add this!
"connect timeout" : 5000
};
and the reconnect function (look in the comments for the explanation)
function reconnect(){
socket.removeAllListeners();
setTimeout(go, 1000);
}
Probably socket.io reuse the same connection without creating a new one, forcing it the app works

Resources