I'm trying to make a communication between nodejs server and a red pitaya card.
The client connect to the web server, use a form to choose some settings , they are send to a red pitaya who send back a big string of numbers.
Then, the web server write them in a downloadable file.
Its working randomly.
here is the server :
var settings = require('./settings_model');
var fs = require('fs');
var net = require('net');
var msg = [];
module.exports = function(app) {
net.createServer(function (socket) {
socket.on('data', function (data) {
/* test console */
var msg = JSON.stringify(data);
console.log("data received :" + msg);
tab = new Array();
for(i = 0; i < nbPix; i++){
tab[i] = new Array();
fs.appendFile('./public/img.txt', '\n');
for(var j=0 ; j < len; j++){
tab[i][j]= data[i+(j*nbPix)];
if(!isNaN(tab[i][j])){
fs.appendFile('./public/img.txt', tab[i][j]+ " ");
};
};
};
});
app.post('/api/sendSettings', function(req, res) {
// creation of the file
fs.writeFile("./public/img.txt", "", function(err){
if(err){console.log(err)};
console.log("the file was saved");
});
// here we send settings to the red pitaya
socket.write(input);
res.end();
});
}).listen(9000);
};
For tiny values, sometimes it works. I can see the log :
data received :{"type":"Buffer","data":[1,1,....2]}
But, for the same values, it can be divided and bug my 2d array
data receive :{"type":"Buffer","data":[1,1,....1]}
data receive :{"type":"Buffer","data":[2,2,....2]}
And if the data received is too big, system crash with an :
Error : EMFILE: too many open files, open './public/img.txt'
Is there a way to set the received stream size ? i think i need all the data to build my 2d array.
For the EMFILE error, ive tried to change settings like "open file" size. Still not working. It should not try to open more than one file anyway ??
i'm new to this kind of project so please if u can give me hints or docs links, ill gladly accept!
Related
I am able to send a Startup Message to the PostgreSQL server I have running and get a response from that server. I get ParameterStatus messages. The problem is I never get any type of Authentication message. My question is this: Why is it that the server never sends any type of Authentication message back to me?
Below I will show you my code snippet for understanding how the startup part of the protocol works, a couple lines of what it outputs for debugging (so that hopefully you won't have to even read my code), what I think is useful information from the PostgreSQL Documentation for understanding my question and another resource I have found to be useful for visualizing the protocol.
This is my code:
var net = require('net');
var BlueBird = require('bluebird');
var Buffer = require('buffer').Buffer;
var createStartupMessage = function(user_name, database_name){
var buffer_size = 22 + user_name.length + 1 + database_name.length + 1 + 1;
var StartUpMessage = new Buffer(buffer_size);
var position_in_buffer = 0;
StartUpMessage.writeUInt32BE(buffer_size, 0);
position_in_buffer += 4;
StartUpMessage.writeUInt32BE(196608, position_in_buffer); //version 3.0
position_in_buffer += 4;
position_in_buffer = addMessageSegment(StartUpMessage, "user", position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, user_name, position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, "database", position_in_buffer);
position_in_buffer = addMessageSegment(StartUpMessage, database_name, position_in_buffer);
//Add the last null terminator to the buffer
addNullTerminatorToMessageSegment(StartUpMessage, position_in_buffer);
console.log("The StartUpMessage looks like this in Hexcode: " + StartUpMessage.toString('hex'));
console.log("The length of the StartupMessage in Hexcode is: " + StartUpMessage.toString('hex').length);
return StartUpMessage;
};
var addMessageSegment = function(StartUpMessage, message_segment, position_in_buffer){
var bytes_in_message_segment = Buffer.byteLength(message_segment);
StartUpMessage.write(message_segment, position_in_buffer, StartUpMessage - position_in_buffer, 'utf8');
position_in_buffer = position_in_buffer + bytes_in_message_segment;
position_in_buffer = addNullTerminatorToMessageSegment(StartUpMessage, position_in_buffer);
return position_in_buffer;
};
var addNullTerminatorToMessageSegment = function(StartUpMessage, position_in_buffer){
StartUpMessage.writeUInt8(0, position_in_buffer);
position_in_buffer = position_in_buffer + 1;
return position_in_buffer;
};
//Here is where everything starts. The functions above are called within this BlueBird Promise.
BlueBird.coroutine(function* () {
var host = "127.0.0.1";
var port = "5432";
var idle_timeout = 10000;
var MySocket = new net.Socket();
MySocket.setTimeout(idle_timeout);
var StartUpMessage = createStartupMessage("testusertwo", "testdatabasetwo");
var data = yield new Promise(
function resolver(resolve, reject) {
var number_of_responses = 0;
var number_of_responses_to_wait_for = 2;
MySocket.on('connect', function () {
var message = StartUpMessage.toString("utf8");
var flushed = MySocket.write(message, "utf8");
console.log("Message flushed to kernel: " + flushed);
});
MySocket.on('data', function (data) {
console.log("The response from the server is: " + data.toString('utf8'));
console.log("----This Line Divides the Response Below from the Response Above----");
if( number_of_responses !== number_of_responses_to_wait_for){
number_of_responses += 1;
} else {
resolve(data);
}
});
MySocket.on('error', function (error) {
reject(error);
});
MySocket.connect(port, host);
}
);
return data;
})()
.then(function (data) {
return data;
})
.catch(function (error) {
console.error(error);
});
These are a couple lines from what my code outputs for debugging purposes. It shows the hexcode representation of the initial utf-8 encoded message I send to the server (startup message format is shown on slide 9 via the link at the bottom). Then it shows the servers response.
After this my program hangs waiting where I am waiting to see it send an Authentication class of message. In the Startup Message I have bolded the first two 32 bit Big Endian integers and all the null terminators for convenience. Also, the ? marks at the end (in ?M2\??ZI) are really those diamond question marks from utf-8 and this ending part changes on every run as well. I do not know why.
Some output from my code:
The StartUpMessage looks like this in Hexcode:
**0000003300030000**75736572**00**746573747573657274776f**00**6461746162617365**00**74657374646174616261736574776f**0000**
The response from the server is:
Sapplication_nameSclient_encodingUTF8SDateStyleISO, MDYSinteger_datetimesonSntervalStylepostgresSis_superuseroffSserver_encodingUTF8Sserver_version9.5.0S&session_authorizationtestusertwoS#standard_conforming_stringsonSTimeZoneUS/EasternK
?M2\??ZI
This is what I think is relevant Information from the Postgresql Documentation:
50.2.Message Flow.1.Start-up:
To begin a session, a frontend opens a connection to the server and sends a startup message.
The authentication cycle ends with the server either rejecting the connection attempt (ErrorResponse), or sending AuthenticationOk.
This section says some other things as well that make it sound like I should either get one of the many Authentication messages listed (such as AuthenticationCleartextPassword message) or an AuthenticationOk if a password is not needed and everything happens without an error. If there is an error, then I should get an ErrorResponse message.
50.5.Message Formats:
In this section it is indicated that if the first Byte in the server response is ’S’, then the Message is classified as a ParameterStatus message.
In this section it also indicates that if the first Byte in the server response is ‘R’, then the Message is classified as an Authentication message.
The useful resource I found:
I think this is a very good resource for visualizing the message flow protocol. The authors name is Jan Urban ́ski. On slide 9, the startup packet is shown. The only thing I’ve found (with node.js anyway) is there needs to be another null terminator box before the . . . box.
https://www.pgcon.org/2014/schedule/attachments/330_postgres-for-the-wire.pdf
After looking on Wireshark, I realized that I was getting an Authentication message ('R' type Message). The problem was that I was parsing the data from the server incorrectly. I immediately converted it to a UTF8 string. The data needs to be parsed according to the message formats before any of it can be converted to UTF8. This is because the formats are not just a bunch of chars strung together. They include 32 bit big endian ints and 16 big endian ints.
This is my server:
[...]
io.sockets.on('connection', function(socket) {
// watching the xml file
fs.watch('../json/notificheJSON.json', function(curr, prev) {
// on file change we can read the new xml
fs.readFile('../json/notificheJSON.json', function(err, data) {
if (err) throw err;
// parsing the new xml data and converting them into json file
var json = JSON.parse(data);
// send the new data to the client
socket.volatile.emit('notification', json);
});
});
});
[...]
This is my client:
[...]
// creating a new websocket
var socket = io.connect('http://localhost:8080');
// on every message recived we print the new datas inside the #container div
socket.on('notification', function (n) {
// convert the json string into a valid javascript object
//var _data = JSON.parse(data);
for(var i = 0; i < n.notifiche.length; ++i) {
if(n.notifiche[i].id_ricevente == <?php echo $id_utente; ?>){
var numero_not = n.notifiche[i].numero_notifiche ;
}
}
$('#notify, #la_mobile').html(numero_not);
});
[...]
Actually, I run my server using the command line:
nodejs server.js
I would like to run my server on remote hosting. How can I do it?
In some form or fashion you need something that will execute node and your application. If you are using remote host that how that works depends on the host, checkout their documentation to see what they say about running node applications.
There are a couple of options for executing node. You can do it using nom scripts. Another option is using an init script, but since you on a remote host I doubt that is the route you want to take unless you are on a VPS.
I am trying to build a simple tcp chat server, WITHOUT socket.io.
Now, I have no problem broadcasting data across all sockets connected to the server.
My problem is assigning a socket identifier to each connection and retrieving them from an object.
Here is the code:
var net = require('net');
//keep track of sockets
var allSockets = {
sockets: {},
addSocket: function(socket, nick, table) {
this.sockets[table+nick] = socket;
},
removeSocket: function(nick, table) {
if (this.sockets[table+nick] !== undefined) {
this.sockets[table+nick] = null;
delete this.sockets[table+nick];
}
}
};
// create the server
var server = net.createServer(function (socket) {
var connected = false;
var jsoncommand = true;
//first data sent MUST BE json formatted string in this format
//{"nick":"someid","table":"tablenumber"}
var thisnick = "";
var thistable = "";
// get client ip
socket.name = socket.remoteAddress;
//write something on each connect
socket.write("You are connecting from " + socket.name + "\n");
socket.write(socket.name + " joined chat\n");
//handle data streams
socket.on('data', function (data) {
if (jsoncommand) {
//JSON.parse the first data stream
var some = JSON.parse(data);
//assign a socket.id based on nick and table
allSockets.addSocket(socket, some.table, some.nick);
socket.write(some.nick + " joined " + some.table + "\n");
thisnick = some.nick;
thistable = some.table;
connected = true;
//no longer waiting for first stream as JSON
jsoncommand = false;
} else if (connected) {
//write whatever data it recieves (function is below)
broadcast(data, thistable);
} else {
socket.write("You are not connected to any table");
socket.destroy();
connected = false;
jsoncommand = true;
thisnick = "";
thistable = "";
}
});
// remove the socket from allSockets but broadcast
//only to other users on the same table
socket.on('end', function () {
allSockets.removeSocket(thisnick, thistable);
broadcast(thisnick + " has left table " + thistable, thistable);
});
//this function should select from the allSockets object,
//only those, whose property matches "table"
//and write to those sockets only, when called
function broadcast(message, table) {
allSockets.sockets.forEach(function(socket) {
if (socket.hasOwnProperty(table)) {
socket.write(message);
}
});
}
});
server.listen(8000);
console.log("running at port 8000\n");
Just deploy this on your machine and connect with nc to port 8000
and be sure that the first thing you send it is something like
{"nick":"mynick","table":"mytable"}
you will see a message that your nick joined your table.
now if you send it something else, based on the fact that it stored your table name,
it should echo whatever you send it, to you and to other connections with different
nicks but on the same table, but the server dies, throwing an error that the allSockets
object, does not have a "for" or "forEach" method or indexOf, or any other.
SO, how do I correct this?
If my nick is "john" and I joined "my_table", and also "mary", "lisa" and "ana" joine the same "my_table", assuming I don't know their nicks, but I do know they are on "my_table",
HOW do I select from the allSockets object, those sockets that contain "my_table".
I tried hasOwnProperty, but that returns boolean, which only tells me that there are sockets with that property, but how do I put them in a for, or foreach loop in order to write to them.
I know it may be a silly question, or maybe im not even aproaching this correctly, but im a node.js beginner, so any advice is greatly apreaciated.
By the way I put this together from examples across the web.
As for the JSON string, its the first thing being sentby a desktop app on connection. Anyways I chose it for testing purposes so don't bother with it.
I suppose error about forEach happens here:
allSockets.sockets.forEach(function(socket) {
While allSockets.sockets is not Array, but it is object (key > value model, like hash table).
So to loop through each socket in it, you should change loop to:
for(var key in allSockets.sockets) {
var socket = allSockets.sockets[key];
// your logic here
}
I'm getting a weird result when writing to a socket. I wrote a simple experiment with a client and a server:
server.js
var net = require('net');
net.createServer(function (connection) {
connection.on('data', function (data) {
console.log('data: ' + data);
});
}).listen(1337);
client.js
var net = require('net');
var client = net.connect({port: 1337}, function () {
var i = 0;
function send() {
client.write('a');
if (++i < 100) {
process.nextTick(send);
} else {
client.end();
}
}
send();
});
I expected the server to show 100 lines of data: a, but I ended up getting a smaller number of data: aaaaaaa lines. There's socket.setNoDelay() that seems to be what I want, but it doesn't seem to have any effect.
What am I missing?
Thanks a lot,
The TCP protocol only sends exactly the bytes you write in the socket. They will not be separated into messages, that's up to you. If you would like to get 100 lines of a then you would have to define 100 separate messages, and choose a delimiter for them. Usually people delimit messages sent to a TCP socket by \r\n.
So you would need to change your server to
var net = require('net');
net.createServer(function (connection) {
connection.on('data', function (buffer) {
var data = buffer.toString();
if (data.indexOf('\r\n') > -1) { // If there's more than one line in the buffer
var lines = data.split('\r\n'); // Split the lines
var i = lines.length;
while (i--) { // This will read your lines in reverse, be careful
console.log(lines[i]); // Print each line
}
} else {
console.log(data); // If only one line came through, print it
}
});
}).listen(1337);
And your client to
var net = require('net');
var client = net.connect({port: 1337}, function () {
var i = 0;
function send() {
client.write('a\r\n'); // Notice the \r\n part. This is what will help you separate messages on the server
if (++i < 100) {
process.nextTick(send);
} else {
client.end();
}
}
send();
});
And then I believe you would get 100 lines of a.
This module also provides a very interesting way to do it, and of course ZeroMQ would also shine in this because it already has a nice protocol that puts things in envelopes and sends them.
Also interestingly but out of the scope of your question, the messages you send write to the socket on one side will not arrive in the same order on the server. If you change your send function to
function send() {
if (++i < 100) {
client.write('a'+i+'\r\n');
process.nextTick(send);
} else {
client.end();
}
}
you can see them arriving not in the order you sent them.
By "The TCP protocol only sends exactly the bytes you write in the socket" I mean that if you do socket.write("1"); socket.write("2"), you will receive "12" on the server, because that's what you wrote on the socket. You have to explicitly separate your messages by something so that the server can know when a message starts and when a message ends.
About receiving things in order or not, you'll notice that if you remove the process.nexTick and have your client like:
var net = require('net');
var client = net.connect({port: 1337}, function () {
var i = 100;
while (i--) {
client.write('a'+i+'\r\n');
}
});
you'll get two messages on the server (at least I got): first numbers 83 - 99 and then 0 - 82, despite having wrote them in order.
Its because TCP splits it in packets in some magic way. The first package was actually larger than the second one, so it got there last. You can read more about how TCP works in the wikipedia page of course, and this video is probably going to say more than what you need to hear but its good to understand everything you're working with.
I have nodejs listening on tcp port and gets content from Flash XMLSocket. If I try to push a lot of data in one message from flash (XMLSocket.send(long_message)) I always end up with event stream.on("data", function(d) { fired while I want it to happen when entire message is transferred.
Flash's XMLSocket transfers data as UTF8 encoded string terminated with null byte.
How can I control my message consistency?
UPDATE
I've found similar question here. But there is no clear answer. I know the end of my message should be null byte, but could you please give me an example on how to store incomplete message and avoid overlapping with next/concurrent message
UPDATE2
After maerics's answer I've done something like
var server = net.createServer(function(stream) {
var dataBlock = "";
stream.on("data", function(d) {
processChunk(d);
});
function processChunk(data) {
var chunks = data.split("\0");
while (chunks.length > 1) {
if (dataBlock.length > 0) {
dataBlock += chunks.shift();
processIncompingMessage(dataBlock);
dataBlock = "";
}
else {
processIncompingMessage(chunks.shift());
}
}
dataBlock += chunks.shift();
}
}
Here's what I would do (tested):
var net = require('net');
var server = net.createServer(function (conn) {
var msg = ''; // Current message, per connection.
conn.setEncoding('utf8');
conn.on('message', function (m) {
console.log('MESSAGE: ' + m);
});
conn.on('data', function (data) {
msg += data.toString('utf8');
if (msg.charCodeAt(msg.length - 1) == 0) {
conn.emit('message', msg.substring(0, msg.length - 1));
msg = '';
}
});
});
Note that it is possible that multiple null separated messages could be encoded in a single data chunk, so you should expand this example to separate the data by null characters and process each one separately. Also, you might want to process the final, potentially incomplete message on the connection 'end' event.