Node net data merge when multiple write - node.js

I am using Node net.
When the client multiple writes feeds, and on the server-side, it received 1 data.
var net = require('net');
// SERVER SIDE
var net_port = 8080; // node net port
var server = net.createServer(function(connection){
connection.on('data', function(data){
var str = data.toString();
console.log("Received, Length: "+str.length); // IT SHOULD be 25 in length
});
connection.pipe(connection);
});
server.listen(net_port, function() {
console.log('server is listening');
});
// SERVER CODE NED
// ### CLIENT SIDE
var client = net.connect({port: 8080}, function() {
console.log('connected to server!');
});
string = "acdefghijklmnopqrstuvwxyz"; // 25 char
setInterval(function(){
client.write(string);
client.write(string);
},1);
setInterval(function(){
client.write(string);
client.write(string);
},1);
Sending string with length 25, but when received OUTPUT IS
Received, Length: 25
Received, Length: 25
Received, Length: 75 // merged 3 data
Received, Length: 100 // merged 4 data
Received, Length: 25
Received, Length: 50 // merge 2 times
Received, Length: 75
Received, Length: 25
Received, Length: 25
Received, Length: 25
Received, Length: 100
IN reality, in my actual code, I cant use split string, because if a client sends String length 100 4 times. so server received
Received, Length: 100
Received, Length: 220
Received, Length: 80
How I can force Node Net to send and received actual data.
Fiddle: https://www.mycompiler.io/view/BJGjA87

net.createServer creates a TCP server. TCP sockets are stream-based, not datagram-based. They don’t have a concept of message boundaries: everything is just a one contiguous stream of bytes with no markers distinguishing which fragments were sent together and which separately. If you want to establish boundaries between messages sent over a TCP socket, you have to delineate them yourself: either by postulating that each message must have a fixed length in bytes, or by introducing markers that establish where one message ends and another starts.
Below, as an example, is a simple modification of your server that uses the json-seq format defined in RFC 7464 to that end. More long-term, you may look into making it a WebSocket server instead, using a library such as ws. It would probably be more robust than the code below; this is mostly just a demonstration.
var net = require('net');
var server = net.createServer(function (connection) {
var buffer = '';
function receivedMessage(value) {
console.log('received: ' + JSON.stringify(value));
}
function protocolError() {
console.log('protocol error');
connection.close();
}
function parseMessages(buf) {
console.log('parsing ' + buf.length + ' code units worth of messages');
if (buf.substring(0, 1) !== '\x1e') {
protocolError();
return;
}
var m, rx = /\x1e([^\x1e]*)/g;
while (m = rx.exec(buffer)) {
if (m[1] === '')
continue;
try {
var value = JSON.parse(m[1]);
} catch (e) {
if (e instanceof SyntaxError)
continue; // RFC 7464 §2.3
throw e;
}
receivedMessage(value);
}
}
connection.on('data', function (data) {
buffer += data.toString('utf8');
console.log(buffer.length + ' code units in the buffer');
var lastRS = buffer.lastIndexOf('\x1e');
if (lastRS === 0)
return;
var fullMessages = buffer.substring(0, lastRS);
buffer = buffer.substring(lastRS, buffer.length);
parseMessages(fullMessages);
try {
var value = JSON.parse(buffer);
} catch (e) {
if (e instanceof SyntaxError)
return; // maybe just truncated; defer error until later
throw e;
}
receivedMessage(value);
buffer = '';
});
connection.on('close', function () {
console.log('closed');
parseMessages(buffer);
});
});
server.listen(8080, function() {
console.log('server is listening');
});
And here’s an appropriately-modified client:
var net = require('net');
var client = net.connect({port: 8080}, function() {
console.log('connected to server!');
});
function sendMessage(value) {
client.write(Buffer.from('\x1e' + JSON.stringify(value) + '\n', 'utf8'));
}
string = 'acdefghijklmnopqrstuvwxyz';
setInterval(function(){
sendMessage(string);
sendMessage(string);
}, 1);
setInterval(function(){
sendMessage(string);
sendMessage(string);
}, 1);

Related

Sending files through net.socket

I trying to create a socket server over the net-module in node.js. There should be different clients connect to it and be able to send/receive commands which will be evaluated on the server in the future.
And also clients should be able to ask for files which the server will send to them.
So, for example, one client is sending the command "file picture.jpg" and then the server will send the content of file picture.jpg through the socket and the client will download it to download.jpg.
This works fine for files <=2kB but not above. It seems like the client gets the files split up in different packages. Which would be fine if I only send files through sockets (So I would write every "package" into the file), but the problem is if I send big files which takes minutes and in that time the server sends other commands (which cant wait for the file transfer to complete).
Here is my simplified code:
server.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
// All connected socket clients will be in this list:
var client = [];
const server = net.createServer(function(sock) {
// We have a connection - a socket object is assigned to the connection automatically
console.log('Client connected: ' + sock.remoteAddress + ':' + sock.remotePort);
// Put this new client in the list
client.push(sock);
sock.on('data', function(data) {
const dataStr = data.toString();
const cmd = dataStr.split(' ', 1)[0];
const value = dataStr.slice(cmd.length).trim();
if (cmd == "file") {
fs.readFile(value, 'utf8', function(err, contents) {
if (err) {
if (err.code == 'ENOENT') {
sock.write('File not exist: '+value); // Send error to client
} else {
console.error(err);
}
} else {
sock.write('file '+contents); // Send file to client
}
});
console.log("Sending file to client.");
} else if (cmd == "quit") {
sock.write('Bye!');
sock.destroy();
} else {
sock.write('Unknown command: "' + dataStr + '"');
}
});
sock.on('error', (e) => {
if(e.code == "ECONNRESET") {
console.log("Got ECONNRESET, continue!");
} else {
console.log(e);
}
});
sock.on('close', function(data) {
var clientId = client.indexOf(sock);
client.splice(clientId, 1);
console.log('Client closed connection: ' + sock.remoteAddress +':'+ sock.remotePort);
});
});
server.on('error', function (e) {
if (e.code == 'EADDRINUSE') {
console.log('Creating server failed: Address in use ' + host +':'+ port);
}
});
server.listen(PORT, HOST, () => {
console.log('Server listening on ' + HOST +':'+ PORT);
});
client.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
var client = new net.Socket();
process.stdin.setEncoding('utf8');
var stdin = process.openStdin();
client.on('error', function (e) {
console.log("ERROR: "+e.code);
process.exit(1);
});
client.connect(PORT, HOST, function() {
console.log('Connected to server: ' + HOST + ':' + PORT);
process.stdout.write('> ');
});
stdin.addListener("data", function(d) {
var inp = d.toString().trim(); // clean input functions here
if (inp === "exit") {
//client.write("goodbye");
client.destroy();
stdin.destroy();
} else {
client.write(inp);
}
});
client.on('data', function(data) {
if (data.toString().substr(0, 5) === 'file ') { // If receiving file from server, data starts with "file "
const filename = "downloaded.jpg";
fs.writeFile(filename, data.toString().substr(5), function(err) {
if(err) {
return console.log(err);
}
console.log("Saved "+filename+".");
process.stdout.write('> ');
});
} else { // Unhandeld commands will be printed on console:
console.log('DATA: ' + data);
process.stdout.write('> ');
}
});
client.on('close', function() {
console.log('Connection closed');
process.exit();
});
So any suggestions how I solve this the best? Also, can I simply expand the buffersize somehow in net.sockets to like 32MB?
Since TCP is emulating a stream, you don't want to rely on anything about how the stream is broken into separate data events. The data given to one callback could be the first half of something or 2 things.
Instead, you want to emulate your datagram protocol on top of this reliable stream by appending stream contents to the end of a buffer and removing complete messages from the front for processing.
For example, this simple server from the docs, demonstrates a minimal valid TCP server:
const net = require('net');
const server = net.createServer((socket) => {
let name = '';
socket.setEncoding('utf8');
socket.on('data', (chunk) => name += chunk);
socket.on('end', () => socket.end(`hello ${name}`));
});
server.listen(8000);
It assembles a buffer with no assumption about the number of data call(s) in its simple case, the buffer is a single message to use at the end event.
To process messages before the end of the connection, you also want to examine the front of the buffer at the end of every data event to look if some messages are complete and ready to process. This separation of complete messages needs to be part of your protocol.
While message separation can be done by length indicators or reserved sequences, reserved sequences require encoding files (to avoid accidentally seeing them in data) and scanning data continuously to find them. This makes length indicators preferable for dealing with the file data.
So for example, the file [data] response first becomes file [#####] [data] where ##### tells you how much data to keep assembling on the buffer before a data callback will remove this entire message from the front of the buffer for processing as a fileSave().
Then, to handle more granular interactivity, simply break up these operations into separate smaller interactions, for example replace file [wholefilecount] [data] responses with filechunk [0-maxchunk] [data] responses that require a filenext command to continue and send a fileend to tell the client the file is done.

NodeJs net.Socket() events not firing

Trying to write a TCP client in Node v0.10.15 and I am having a little trouble getting data back from the server. I know that the server is working properly because I have 3-4 different clients written in different languages communicating with it.
Below is a snippet of a larger piece of code but this should get the point across.
The problem is: I'm expecting 2 packets coming back after writing to the socket (this part is not included in this example). I'm only seeing the "data" event being fired once. Is there something that I need to do to get node to resume reading from the Tcp stream? I can confirm that the server is sending 2 packets(The length and then the actual data) Any help would be appreciated.
var dc = require('./DataContracts.js');
var net = require('net');
require('buffertools').extend();
var client = net.Socket();
var isConnected = false;
var serverHost = '10.2.2.21';
var dataCallback;
var receivedBuffer = new Array();
function InitComm(buffer) {
if (!isConnected) {
client.connect(4987, serverHost, function() {
client.on('data', function(data) {
console.log('Received server packet...');
var buf = new Buffer(data);
receivedBuffer.push(buf);
client.resume();
});
client.on('end', function() {
if (receivedBuffer.length > 1) {
if (dataCallback !== undefined)
dataCallback(receivedBuffer);
}
});
client.on('close', function() {
//clean up
});
client.on('error', function(err) {
console.log('Error!: ' + err);
});
Communicate(buffer);
});
} else {
Communicate(buffer);
}
}
Turns out that node was combining both of the packets together. I must be missing a Carriage return on the first packet

nodejs tcp listener script

I have the following situation and just wanted to check if I am doing it right. I have a couple of devices at my customers (RS232). Now I have a RS232-WIFI dongle connected with it, so data out of the device is sent over to my server (it outputs a datastring, example: {12,1,etc). On my server I have a NodeJS script running, that opens up a port and fetches all data coming in.
var net = require('net');
var host = '1.1.1.1';
var servers = [];
var ports = [20000, 20001, 20002, 20003, 20004];
// Create servers
ports.forEach(function (port) {
var s = net.createServer(function (sock) {
// We have a connection - a socket object is assigned to the connection automatically
console.log('CONNECTED (' + sock.localPort + '): ' + sock.remoteAddress + ':' + sock.remotePort);
// Add a 'data' event handler to this instance of socket
sock.on('data', function (data) {
// post data to a server so it can be saved and stuff
postData(data.toString(), sock);
// close connection
sock.end();
});
sock.on('error', function (error) {
console.log('******* ERROR ' + error + ' *******');
// close connection
sock.end();
});
});
s.listen(port, host, function () {
console.log('Server listening on ' + host + ':' + s.address().port);
});
servers.push(s);
});
Okay, so this works pretty good. But I am wondering, sometimes not all of the data is posted at once, sometimes I get {12, and after a second I get the rest (or even more times is needed). What can I do to optimize this script further? Do I need to call sock.end(); after receiving data? Does this hurt network performance for me or my customers?
If you guys need more info let me know.
That depends on the protocol of your devices, if the devices use each connection for a chunk of data, you can write the program like so: (Do not close the socket on data event)
....
// socket will close and destroy automatically after the device close the connection
var s = net.createServer(function (sock) {
sock.setEncoding('utf8');
var body = "";
sock.on('data', function (data) {
body = body + data;
});
sock.on('end', function() {
console.log(data);
postData(data);
});
// TODO error handling here
});
....
Note: Socket is not guaranteed to give you all data at once, you should listen data event then concat all chunks before using.
If your devices don't close socket, you will not receive on('end'), then the code should be like this:
....
var s = net.createServer(function (sock) {
sock.setEncoding('utf8');
// var body = "";
sock.on('data', function (data) {
// body = body + data;
postData(data);
});
sock.on('end', function() {
console.log('end');
});
// TODO error handling here
});
....

EMFILE error when using nodejs redis

I want to test how many connections my redis server can hold, so I call redis.createClient() in a loop, while the redis server still runs lively, I got the EMFILE error, I know that I have used out my fds.
but wait, I have just test my mqtt server before, I did the same thing to my mqtt server,
I called mqtt.createClient() in a loop of 10000, of 20000... but I never got the EMFILE error .
so, does the nodejs mqtt library use a different mechanism underneath?
redis-client.js :
var redis = require('redis');
function start() {
var client = redis.createClient();
client.on('error', function(err) {
console.log('Error ' + err);
});
}
exports.start = start;
redis-test.js
var redis_client = require('./redis-client');
for(var i = 0 ; i < 10000 ; ++i) {
redis_client.start();
console.log('redis client ' + i + ' started');
}
mqtt-subclient.js
var mqtt = require('mqtt');
function start() {
var client = mqtt.createClient();
client.subscribe('message');
//client.publish('message', 'hello me!');
client.on('message', function(topic, message) {
console.log('receive message: ');
console.log(message);
});
client.on('connack', function(packet) {
console.log(packet);
if(packet.returnCode == 0) {
console.log('connect successfully');
}
});
client.on('suback', function(packet) {
console.log(packet.messageId);
});
client.on('error', function(err) {
console.log(err);
});
}
exports.start = start;
mqtt-test.js
var subclient = require('./mqtt-subclient.js');
for(var i = 0 ; i < 10000 ; ++i) {
subclient.start();
console.log('client ' + i + ' started');
}
Redis cannot accept more than x simultaneous connection attempts, where x is the backlog parameter of the listen system call.
It is limited by the minimum between the somaxconn kernel parameter (128 is a common default value), and 512. So if you attempt more than min(somaxconn,512) simultaneous connections, you can have errors. If you add a small delay between your connection attempts, it should fix this problem.
Then, you need to check that you have enough resources to open 10000 file descriptors (check the output of ulimit -a), and that your TCP/IP ephemeral port range is big enough to accomodate such a number of client connections.

Reading a clients data input in a NodeJS TCP Server

Im trying to create a NodeJS TCP Server, that will read to the clients input and then act accordingly.
I'd like to know how I can read the data, so I can set up conditionals to perform process.
var net = require('net');
var server = net.createServer(function (socket) {
socket.on('data', function(data) {
buf = new Buffer(256);
len = buf.write(data.toString());
if (buf.toString('utf8', 0, len) === "test"){
console.log("you typed test");
}
console.log(len + " bytes: " + buf.toString('utf8', 0, len));
});
socket.write("Connected to server.\r\n");
});
server.listen(8080, "127.0.0.1");
I am outputting the value inputted here :
console.log(len + " bytes: " + buf.toString('utf8', 0, len));
but my if statement above this log, isnt matching the value 'test' when I actually type 'test' in the client terminal window.
Any help is appreciated
-chris
I worked it out using the toString() method:
socket.on('data', function(data) {
var response = data.toString().trim();
if (/disconnect/.test(response)) {
console.log("Client is diconnecting.");
socket.end('Disconnecting you now.\r\n');
}
});
socket.on('end', function() {
console.log('client disconnected');
});

Resources