I am wondering if it makes sense to use Node's Buffer for incoming client data to a server. My server and clients are TCP based and I am using <EOF> to determine the end of a message. The message is always stringified JSON.
eg: {"name":"Bob"}<EOF>
In case the entire message does not come through, should I be using Node Buffer to build up the data, or a regular string?
If it is a buffer, I don't think I understand how to correctly build one up. This is my Client constructor that is created each time a socket connects to the Node server.
constructor(socket){
var self = this;
// properties
this.socket = socket;
this.buffer = Buffer.alloc(1024);
this.dataEnd = '<EOF>';
// append <EOF> to every msg
this.socket.write = function(msg){
msg += "<EOF>";
return Socket.prototype.write.call(this, msg);
};
// build up buffer
this.socket.on('data', function(data){
var buffLen = self.buffer.length;
var dataBuffer = Buffer.from(data);
if(buffLen + dataBuffer.length < 1024){
if(data.indexOf(self.dataEnd) === -1){
self.buffer.concat(dataBuffer);
}
}
});
return this;
}
Related
I am able to send a Startup Message to the PostgreSQL server I have running and get a response from that server. I get ParameterStatus messages. The problem is I never get any type of Authentication message. My question is this: Why is it that the server never sends any type of Authentication message back to me?
Below I will show you my code snippet for understanding how the startup part of the protocol works, a couple lines of what it outputs for debugging (so that hopefully you won't have to even read my code), what I think is useful information from the PostgreSQL Documentation for understanding my question and another resource I have found to be useful for visualizing the protocol.
This is my code:
var net = require('net');
var BlueBird = require('bluebird');
var Buffer = require('buffer').Buffer;
var createStartupMessage = function(user_name, database_name){
var buffer_size = 22 + user_name.length + 1 + database_name.length + 1 + 1;
var StartUpMessage = new Buffer(buffer_size);
var position_in_buffer = 0;
StartUpMessage.writeUInt32BE(buffer_size, 0);
position_in_buffer += 4;
StartUpMessage.writeUInt32BE(196608, position_in_buffer); //version 3.0
position_in_buffer += 4;
position_in_buffer = addMessageSegment(StartUpMessage, "user", position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, user_name, position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, "database", position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, database_name, position_in_buffer);
//Add the last null terminator to the buffer
addNullTerminatorToMessageSegment(StartUpMessage, position_in_buffer);
console.log("The StartUpMessage looks like this in Hexcode: " + StartUpMessage.toString('hex'));
console.log("The length of the StartupMessage in Hexcode is: " + StartUpMessage.toString('hex').length);
return StartUpMessage;
};
var addMessageSegment = function(StartUpMessage, message_segment, position_in_buffer){
var bytes_in_message_segment = Buffer.byteLength(message_segment);
StartUpMessage.write(message_segment, position_in_buffer, StartUpMessage - position_in_buffer, 'utf8');
position_in_buffer = position_in_buffer + bytes_in_message_segment;
position_in_buffer = addNullTerminatorToMessageSegment(StartUpMessage, position_in_buffer);
return position_in_buffer;
};
var addNullTerminatorToMessageSegment = function(StartUpMessage, position_in_buffer){
StartUpMessage.writeUInt8(0, position_in_buffer);
position_in_buffer = position_in_buffer + 1;
return position_in_buffer;
};
//Here is where everything starts. The functions above are called within this BlueBird Promise.
BlueBird.coroutine(function* () {
var host = "127.0.0.1";
var port = "5432";
var idle_timeout = 10000;
var MySocket = new net.Socket();
MySocket.setTimeout(idle_timeout);
var StartUpMessage = createStartupMessage("testusertwo", "testdatabasetwo");
var data = yield new Promise(
function resolver(resolve, reject) {
var number_of_responses = 0;
var number_of_responses_to_wait_for = 2;
MySocket.on('connect', function () {
var message = StartUpMessage.toString("utf8");
var flushed = MySocket.write(message, "utf8");
console.log("Message flushed to kernel: " + flushed);
});
MySocket.on('data', function (data) {
console.log("The response from the server is: " + data.toString('utf8'));
console.log("----This Line Divides the Response Below from the Response Above----");
if( number_of_responses !== number_of_responses_to_wait_for){
number_of_responses += 1;
} else {
resolve(data);
}
});
MySocket.on('error', function (error) {
reject(error);
});
MySocket.connect(port, host);
}
);
return data;
})()
.then(function (data) {
return data;
})
.catch(function (error) {
console.error(error);
});
These are a couple lines from what my code outputs for debugging purposes. It shows the hexcode representation of the initial utf-8 encoded message I send to the server (startup message format is shown on slide 9 via the link at the bottom). Then it shows the servers response.
After this my program hangs waiting where I am waiting to see it send an Authentication class of message. In the Startup Message I have bolded the first two 32 bit Big Endian integers and all the null terminators for convenience. Also, the ? marks at the end (in ?M2\??ZI) are really those diamond question marks from utf-8 and this ending part changes on every run as well. I do not know why.
Some output from my code:
The StartUpMessage looks like this in Hexcode:
**0000003300030000**75736572**00**746573747573657274776f**00**6461746162617365**00**74657374646174616261736574776f**0000**
The response from the server is:
Sapplication_nameSclient_encodingUTF8SDateStyleISO, MDYSinteger_datetimesonSntervalStylepostgresSis_superuseroffSserver_encodingUTF8Sserver_version9.5.0S&session_authorizationtestusertwoS#standard_conforming_stringsonSTimeZoneUS/EasternK
?M2\??ZI
This is what I think is relevant Information from the Postgresql Documentation:
50.2.Message Flow.1.Start-up:
To begin a session, a frontend opens a connection to the server and sends a startup message.
The authentication cycle ends with the server either rejecting the connection attempt (ErrorResponse), or sending AuthenticationOk.
This section says some other things as well that make it sound like I should either get one of the many Authentication messages listed (such as AuthenticationCleartextPassword message) or an AuthenticationOk if a password is not needed and everything happens without an error. If there is an error, then I should get an ErrorResponse message.
50.5.Message Formats:
In this section it is indicated that if the first Byte in the server response is ’S’, then the Message is classified as a ParameterStatus message.
In this section it also indicates that if the first Byte in the server response is ‘R’, then the Message is classified as an Authentication message.
The useful resource I found:
I think this is a very good resource for visualizing the message flow protocol. The authors name is Jan Urban ́ski. On slide 9, the startup packet is shown. The only thing I’ve found (with node.js anyway) is there needs to be another null terminator box before the . . . box.
https://www.pgcon.org/2014/schedule/attachments/330_postgres-for-the-wire.pdf
After looking on Wireshark, I realized that I was getting an Authentication message ('R' type Message). The problem was that I was parsing the data from the server incorrectly. I immediately converted it to a UTF8 string. The data needs to be parsed according to the message formats before any of it can be converted to UTF8. This is because the formats are not just a bunch of chars strung together. They include 32 bit big endian ints and 16 big endian ints.
I have code like that
var subscribeNewMessages = require("redis").createClient(config.redis.port, config.redis.host);
subscribeNewMessages.subscribe('new-messages');
io.of('/new-messages').on('connection', function (client) {
subscribeNewMessages.on("message", function(channel, message) {
var obj = JSON.parse(message);
if (client.userId == obj.toUserId || client.guestId == obj.toUserId) {
client.send(message);
}
obj = null;
});
})
And how can I optimize it? Because this code parses string json for each new messages.
Also when I try to publish to redis chanel I need to JSON.stringify
redis1.publish(channelPrefix, JSON.stringify(clientData));
There isn't going to be a way to avoid JSON.parse()/JSON.stringify() as long as you're storing js objects. You could use a different serialization format like msgpack, but you're still calling functions to serialize/unserialize your data (also JSON.parse()/JSON.stringify() are already pretty hard to beat performance-wise in node).
I think the only real performance adjustment you could make with the code you've provided is to only parse the JSON once for all clients instead of for each client. Example:
var subscribeNewMessages = require('redis').createClient(config.redis.port, config.redis.host);
subscribeNewMessages.subscribe('new-messages');
var nsNewMsgs = io.of('/new-messages');
subscribeNewMessages.on('message', function(channel, message) {
var obj = JSON.parse(message),
clients = nsNewMsgs.connected,
ids = Object.keys(clients);
for (var i = 0, len = ids.length, client; i < len; ++i) {
client = clients[ids[i]];
if (client.userId == obj.toUserId || client.guestId == obj.toUserId)
client.send(message);
}
});
Depending on your application, you might even be able to avoid the for-loop entirely if you can store the socket.id values in your published messages, then you can simply look up clients[obj.userSockId] and clients[obj.guestSockId] because the connected is keyed on socket.id.
Trying to send data from a serial device to web clients. I am using a serial to network proxy, ser2Net to make the data available to a server that acts on the data and sends a manipulated version of the data to web clients. The clients specify the location of the ser2net host and port. The core of this action is coded in node.js as shown here:
function getDataStream(socket, dataSourcePort, host) {
var dataStream = net.createConnection(dataSourcePort, host),
dataLine = "";
dataStream.on('error', function(error){
socket.emit('error',{message:"Source not found on host:"+ host + " port:"+dataSourcePort});
console.log(error);
});
dataStream.on('connect', function(){
socket.emit('connected',{message:"Data Source Found"});
});
dataStream.on('close', function(){
console.log("Close socket");
});
dataStream.on('end',function(){
console.log('socket ended');
dataConnection.emit('lost',{connectInfo:{host:host,port:dataSourcePort}});
});
dataStream.on('data', function(data) {
// Collect a line from the host
line += data.toString();
// Split collected data by delimiter
line.split(delimiter).forEach(function (part, i, array) {
if (i !== array.length-1) { // Fully delimited line.
//push on to buffer and emit when bufferSendCommand is present
dataLine = part.trim();
buffer.push(part.trim());
if(part.substring(0, bufferSendCommand.length) == bufferSendCommand){
gotALine.emit('new', buffer);
buffer=[];
}
}
else {
// Last split part might be partial. We can't announce it just yet.
line = part;
}
});
});
return dataStream;
}
io.sockets.on('connection', function(socket){
var stream = getDataStream(socket, dataSourcePort, host);
//dispense incoming data from data server
gotALine.on('new', function(buffer){
socket.emit('feed', {feedLines: buffer});
});
dataConnection.on('lost', function(connectInfo){
setTimeout(function(){
console.log("Trying --- to reconnect ");
stream = getDataStream(socket, connectInfo.port, connectInfo.host);
},5000);
});
// Handle Client request to change stream
socket.on('message',function(data) {
var clientMessage = JSON.parse(data);
if('connectString' in clientMessage
&& clientMessage.connectString.dataHost !== ''
&& clientMessage.connectString.dataPort !== '') {
stream.destroy();
stream = getDataStream(socket,
clientMessage.connectString.dataPort,
clientMessage.connectString.dataHost);
}
});
});
This works well enough until the serial device drops off and ser2net stops sending data. My attempt to catch the end of the socket and reconnect is not working. The event gets emitted properly but the setTimeout only goes once. I would like to find a way to keep on trying to reconnect while sending a message to the client informing or retry attempts. I am node.js newbie and this may not be the best way to do this. Any suggestions would be appreciated.
Ok I think I figured it out in the dataStream.on('data' ... I added a setTimeout
clearTimeout(connectionMonitor);
connectionMonitor = setTimeout(function(){doReconnect(socket);}, someThresholdTime);
The timeout executes if data stops coming in, as it is repeatedly cleared each time data comes in. The doReconnect function keeps trying to connect and sends a message to the client saying something bad is going on.
I am trying to build a simple tcp chat server, WITHOUT socket.io.
Now, I have no problem broadcasting data across all sockets connected to the server.
My problem is assigning a socket identifier to each connection and retrieving them from an object.
Here is the code:
var net = require('net');
//keep track of sockets
var allSockets = {
sockets: {},
addSocket: function(socket, nick, table) {
this.sockets[table+nick] = socket;
},
removeSocket: function(nick, table) {
if (this.sockets[table+nick] !== undefined) {
this.sockets[table+nick] = null;
delete this.sockets[table+nick];
}
}
};
// create the server
var server = net.createServer(function (socket) {
var connected = false;
var jsoncommand = true;
//first data sent MUST BE json formatted string in this format
//{"nick":"someid","table":"tablenumber"}
var thisnick = "";
var thistable = "";
// get client ip
socket.name = socket.remoteAddress;
//write something on each connect
socket.write("You are connecting from " + socket.name + "\n");
socket.write(socket.name + " joined chat\n");
//handle data streams
socket.on('data', function (data) {
if (jsoncommand) {
//JSON.parse the first data stream
var some = JSON.parse(data);
//assign a socket.id based on nick and table
allSockets.addSocket(socket, some.table, some.nick);
socket.write(some.nick + " joined " + some.table + "\n");
thisnick = some.nick;
thistable = some.table;
connected = true;
//no longer waiting for first stream as JSON
jsoncommand = false;
} else if (connected) {
//write whatever data it recieves (function is below)
broadcast(data, thistable);
} else {
socket.write("You are not connected to any table");
socket.destroy();
connected = false;
jsoncommand = true;
thisnick = "";
thistable = "";
}
});
// remove the socket from allSockets but broadcast
//only to other users on the same table
socket.on('end', function () {
allSockets.removeSocket(thisnick, thistable);
broadcast(thisnick + " has left table " + thistable, thistable);
});
//this function should select from the allSockets object,
//only those, whose property matches "table"
//and write to those sockets only, when called
function broadcast(message, table) {
allSockets.sockets.forEach(function(socket) {
if (socket.hasOwnProperty(table)) {
socket.write(message);
}
});
}
});
server.listen(8000);
console.log("running at port 8000\n");
Just deploy this on your machine and connect with nc to port 8000
and be sure that the first thing you send it is something like
{"nick":"mynick","table":"mytable"}
you will see a message that your nick joined your table.
now if you send it something else, based on the fact that it stored your table name,
it should echo whatever you send it, to you and to other connections with different
nicks but on the same table, but the server dies, throwing an error that the allSockets
object, does not have a "for" or "forEach" method or indexOf, or any other.
SO, how do I correct this?
If my nick is "john" and I joined "my_table", and also "mary", "lisa" and "ana" joine the same "my_table", assuming I don't know their nicks, but I do know they are on "my_table",
HOW do I select from the allSockets object, those sockets that contain "my_table".
I tried hasOwnProperty, but that returns boolean, which only tells me that there are sockets with that property, but how do I put them in a for, or foreach loop in order to write to them.
I know it may be a silly question, or maybe im not even aproaching this correctly, but im a node.js beginner, so any advice is greatly apreaciated.
By the way I put this together from examples across the web.
As for the JSON string, its the first thing being sentby a desktop app on connection. Anyways I chose it for testing purposes so don't bother with it.
I suppose error about forEach happens here:
allSockets.sockets.forEach(function(socket) {
While allSockets.sockets is not Array, but it is object (key > value model, like hash table).
So to loop through each socket in it, you should change loop to:
for(var key in allSockets.sockets) {
var socket = allSockets.sockets[key];
// your logic here
}
I have nodejs listening on tcp port and gets content from Flash XMLSocket. If I try to push a lot of data in one message from flash (XMLSocket.send(long_message)) I always end up with event stream.on("data", function(d) { fired while I want it to happen when entire message is transferred.
Flash's XMLSocket transfers data as UTF8 encoded string terminated with null byte.
How can I control my message consistency?
UPDATE
I've found similar question here. But there is no clear answer. I know the end of my message should be null byte, but could you please give me an example on how to store incomplete message and avoid overlapping with next/concurrent message
UPDATE2
After maerics's answer I've done something like
var server = net.createServer(function(stream) {
var dataBlock = "";
stream.on("data", function(d) {
processChunk(d);
});
function processChunk(data) {
var chunks = data.split("\0");
while (chunks.length > 1) {
if (dataBlock.length > 0) {
dataBlock += chunks.shift();
processIncompingMessage(dataBlock);
dataBlock = "";
}
else {
processIncompingMessage(chunks.shift());
}
}
dataBlock += chunks.shift();
}
}
Here's what I would do (tested):
var net = require('net');
var server = net.createServer(function (conn) {
var msg = ''; // Current message, per connection.
conn.setEncoding('utf8');
conn.on('message', function (m) {
console.log('MESSAGE: ' + m);
});
conn.on('data', function (data) {
msg += data.toString('utf8');
if (msg.charCodeAt(msg.length - 1) == 0) {
conn.emit('message', msg.substring(0, msg.length - 1));
msg = '';
}
});
});
Note that it is possible that multiple null separated messages could be encoded in a single data chunk, so you should expand this example to separate the data by null characters and process each one separately. Also, you might want to process the final, potentially incomplete message on the connection 'end' event.