WebSocket stops receiving data after 15 - 20 minutes of data stream - NodeJS - node.js

Question

Code
var websock = net.createServer(function(sock) {
sock.pipe(sock);
sock.setEncoding('utf8');
sock.setKeepAlive(true);
sock.on("data", function(d) {
console.log("websock", d);
});
sock.on('end', function() {
console.log('websock disconnected');
});
});
websock.listen(777, '127.0.0.1');
After few minutes ~15 mins the callback code in sock.on("data", function() {}) seems not to be working. why is it the case? I checked the console.log, there is no log with a string "websock disconnected".
if the socket is not disconnected and if there is no error, what has happened to the socket connection or the data stream?
On the other end, (Server side, data sender) seems to be streaming data continuously while client side (nodejs app) has stopped receiving data.

Answer 1

The issue arises from your use of the pipe mechanism to echo back data which is never consumed on the original side (communication is unidirectional):
sock.pipe(sock);
This makes your code work as an echo server. Your socket "sock" is a duplex stream (i.e. both readable - for the incoming data you receive, and writable - for outgoing data you send back).
A quick fix if you don't need to respond back and you just need to receive data is to simply delete the "sock.pipe(sock);" line. To find out the explanation, read ahead.
Most probably your data source (the MT5 application you mentioned) sends data continuously and it doesn't read what you send back at all. So, your code keeps echoing back the received data using sock.pipe(sock), filling the outgoing buffer which is never consumed. However, the pipe mechanism of Nodejs streams handles backpressure, which means that when two streams (a readable and a writable one) are connected by a pipe, if the outgoing buffer is filling (reaching a high watermark), the readable stream is paused, to prevent the "overflow" of the writable stream.
You can read more about backpressure in the Nodejs docs. This fragment particularly describes how streams are handling backpressure:
In Node.js the source is a Readable stream and the consumer is the Writable stream [...]
The moment that backpressure is triggered can be narrowed exactly to the return value of a Writable's .write() function. [...]
In any scenario where the data buffer has exceeded the highWaterMark or the write queue is currently busy, .write() will return false.
When a false value is returned, the backpressure system kicks in. It will pause the incoming Readable stream from sending any data and wait until the consumer is ready again.
Below you can find my setup to show where backpressure kicks in; there are two files, server.js and client.js. If you run them both, server will write to console "BACKPRESSURE" soon. As the server is not handling backpressure (it ignores that sock.write starts returning false at some point), the outgoing buffer is filled and filled, consuming more memory, while in your scenario, socket.pipe was handling backpressure and thus it paused the flow of the incoming messages.
The server:
// ----------------------------------------
// server.js
var net = require('net');
var server = net.createServer(function (socket) {
console.log('new connection');
// socket.pipe(socket); // replaced with socket.write on each 'data' event
socket.setEncoding('utf8');
socket.setKeepAlive(true);
socket.on("data", function (d) {
console.log("received: ", d);
var result = socket.write(d);
console.log(result ? 'write ok' : 'BACKPRESSURE');
});
socket.on('error', function (err) {
console.log('client error:', err);
});
socket.on('end', function () {
console.log('client disconnected');
});
});
server.listen(10777, '127.0.0.1', () => {
console.log('server listening...');
});
The client:
// ----------------------------------------
// client.js
var net = require('net');
var client = net.createConnection(10777, () => {
console.log('connected to server!' + new Date().toISOString());
var count = 1;
var date;
while(count < 35000) {
count++;
date = new Date().toISOString() + '_' + count;
console.log('sending: ', date);
client.write(date + '\n');
}
});
client.on('data', (data) => {
console.log('received:', data.toString());
});
client.on('end', () => {
console.log('disconnected from server');
});