const net = require('net')
const sockets = []
server.on('connection', sock => {
log("tcp_server", "info", `Connected at ${sock.remoteAddress}:${sock.remotePort}`)
sockets.push(sock);
// Write the data back to all the connected, the client will receive it as data from the server
sockets.forEach((sock, index, array) => {
})
sock.on('data', data => {
})
// Add a 'close' event handler to this instance of socket
sock.on('close', data => {
}) // end sock.on
})
server.listen(conf.port, conf.serverHost, () => {
const address = server.address()
const port = address.port
const family = address.family
const ipaddr = address.address
log("tcp_server", "info", 'Server is listening at port ' + port)
log("tcp_server", "info", 'Server ip :' + ipaddr)
log("tcp_server", "info", 'Server is IP4/IP6 : ' + family)
})
this is my socket tcp server
and i faced problem with receive chunk data in c# its easy to receive every chunk if i know the size of it
and i can control how much byte i want to receive every time
now in NodeJs i can get the buffer length by get first 5 byte
and then get the size of the message
var size = (buff[1] << 24) | (buff[2] << 16) | (buff[3] << 8) | buff[4];
i tried this simple code for get the chunk data and process it
first i define a Buffer in top
var mybuffer = Buffer.alloc(30);
var length = 0;
sock.on('data', data => {
data.copy(mybuffer, length , 0, data.length); //copy buffer to mybuffer
length += data.length;
//now check if my length is 5 or greater i can determine the buffer size i sent
size = (buff[1] << 24) | (buff[2] << 16) | (buff[3] << 8) | buff[4];
//now need to continue receive until i reach (size - length) = total bytes i sent
})
this what i tried for receive all the chunk but still need more work
any idea how to receive all data depend on size i sent on first 5 bytes of every message
I found this code easysocket! similar to what i want i just made simple edit to the code and its work like charm
Related
I'm using Node's TCP client for a real-time backend for my game, which communicates in a fixed-length protocol. For example,
0x00 - Connection Request
int32 Identifier
4 bytes long
0x01 - Position Update
int32 positionx
int32 positiony
8 bytes long
Of course, Node works like this
socket.on("data", (data) => {
// Arbitary buffer of bytes
})
I only want to process packets one at a time once they've been recieved in full, but what's the best way to A: continue adding to a buffer until the full packet is recieved and B: make sure not to include the data of a second packet in the first one
This is how I solved it!
function readData(data, socket) {
console.log("Read chunk of data " + data.length + " bytes long")
while (data.length > 0) {
if (expectsNewPacket) {
packetBuffer = new Buffer(0)
currentPacketInfo = getPacketInformationFromHeader(data)
data = currentPacketInfo.slicedData
console.log("New packet: ident " + currentPacketInfo.ident + ", length: " + currentPacketInfo.length)
expectedLengthRemaining = currentPacketInfo.length
expectsNewPacket = false
}
// If the data is more than the rest of the packet, just concat the rest of the packet and remove it from our block
if (data.length >= expectedLengthRemaining) {
packetBuffer = Buffer.concat([packetBuffer, data.slice(0, expectedLengthRemaining)])
data = data.slice(expectedLengthRemaining)
processPacket(packetBuffer, socket)
expectsNewPacket = true
} else {
// Or if the data length is less than what we need, just add all that we can and we'll add more later
packetBuffer = Buffer.concat([packetBuffer, data.slice(0, data.length)])
data = data.slice(data.length)
}
}
}
I'm now trying to send bytes continuously from node.js(server) to Android(client). Let me show the code example.
var net = require('net');
var server = net.createServer(function(c){
c.on('data', function(data){
if(data == 'foo'){
for(var i = 1; i <= 255; i++){
var byteData = makeBytedata();
c.write(byteData);
wait(100)
}
}
});
});
This code does not works fine because it sometimes combines byteData to one packet. Does anyone have solution to send bytes separately?
net.createServer create TCP server, TCP does not send messages separately. TCP is a stream protocol, which means that when you write bytes to the socket, you get the same bytes in the same order at the receiving end.
One work around: define a format for your message, so that your client can determine the beginning and end of a message within the socket stream. For example, you could use a \n to mark the end of a message.
for(var i = 1; i <= 255; i++){
var byteData = makeBytedata();
c.write(byteData + '\n');
}
Then the client could separate them by \n.
The other way could be to use UDP/Dgram
var dgram = require("dgram"),
server = dgram.createSocket('udp4');
server.on("message", function(msg, rinfo) {
// send message to client
});
I'm trying to build the back half of a Paranoid Pirate Pattern, a ROUTER that sends work out to a set of DEALER nodes (it's possible that I'm misundertanding the diagram). For now I just want the DEALERs to echo the work back or just send back a message that says "done". The problem is that the worker node (DEALER) is never receiving any messages.
var buildSocket, connectionTemplate, delay, frontPort, log, q, qPort, worker, workerPort, zmq;
zmq = require("zmq");
frontPort = 5000;
qPort = 5100;
workerPort = 5200;
connectionTemplate = "tcp://127.0.0.1:";
log = console.log;
debugger;
delay = process.argv[2] || 1000;
buildSocket = function(desc, socketType, port) {
var socket;
log("creating " + socketType + " socket");
socket = zmq.socket(socketType);
socket.identity = "" + desc + "-" + socketType + "-" + process.pid + "-" + port;
return socket;
};
q = buildSocket('q_output', 'router', qPort);
worker = buildSocket('worker', 'dealer', workerPort);
worker.bind("" + connectionTemplate + workerPort);
q.connect("" + connectionTemplate + workerPort);
q.on('message', function() {
var args;
args = Array.apply(null, arguments);
log('queue received ' + JSON.stringify(arguments));
return worker.send('work done');
});
worker.on('message', function() {
var args;
log('back received ' + JSON.stringify(arguments));
args = Array.apply(null, arguments);
return q.send(args);
});
setInterval((function() {
var value;
value = Math.floor(Math.random() * 100);
console.log(q.identity + ": sending " + value);
q.send(value);
}), delay);
The queue and worker on 'message' events never fire. The way I understand this is you set up the ROUTER node, bind it to a port (for return messages), set up the DEALER nodes and bind them to a port then connect the ROUTER to the DEALER port and start sending messages. In practice, messages are sent but never received:
creating router socket
creating dealer socket
q_output-router-60326-5100: sending 30
q_output-router-60326-5100: sending 25
q_output-router-60326-5100: sending 65
q_output-router-60326-5100: sending 68
q_output-router-60326-5100: sending 50
q_output-router-60326-5100: sending 88
You've got things a little backwards, here. Think of a DEALER socket as a modified REQ socket... it should be initiating your messages to your router. A ROUTER socket is more like a modified REP socket... it should be responding to the initial request sent by your dealer.
You don't strictly need to follow that pattern with ROUTER/DEALER pairings... but it definitely makes things much easier, so you should stick with it while you're learning.
The second thing that sticks out to me from your code is that you message handlers, you've got the wrong socket sending messages.
Take for instance this code (directly copied without modification):
q.on('message', function() {
var args;
args = Array.apply(null, arguments);
log('queue received ' + JSON.stringify(arguments));
return worker.send('work done');
});
... that says (in psuedocode):
when `q` receives a message from `worker`
print out the message we received
now have `worker` send *another* message that says "work done"
What you want is something more like the following (simplified):
var zmq = require("zmq");
var q = zmq.socket('router');
var worker = zmq.socket('dealer');
// I've switched it so the router is binding and the worker is connecting
// this is somewhat arbitrary, but generally I'd consider workers to be less
// reliable, more transient, and also more numerous. I'd think of the queue
// as the "server"
// I've used bindSync, which is synchronous, but that's typically OK in the
// startup phase of a script, and it simplifies things. If you're spinning
// up new sockets in the middle of your script, using the async bind()
// is more appropriate
q.bindSync('tcp://127.0.0.1:5200');
worker.connect('tcp://127.0.0.1:5200');
q.on('message', function() {
var args;
args = Array.apply(null, arguments);
log('queue received ' + JSON.stringify(arguments));
// if we've received a message at our queue, we know the worker is ready for
// more work, so we ready some new data, regardless of whether we
// received work back
var value = Math.floor(Math.random() * 100);
// note that the socket that received the message is responding back
if (args[1].toString() == 'ready') console.log('worker now online');
else console.log('work received: '+args[1].toString());
// we need to specify the "ID" of our worker as the first frame of
// the message
q.send([args[0], value]);
// we don't need to return anything, the return value of a
// callback is ignored
});
worker.on('message', function() {
var args;
log('back received ' + JSON.stringify(arguments));
args = Array.apply(null, arguments);
// we're just echoing back the "work" we received from the queue
// for additional "workiness", we wait somewhere between 10-1000
// milliseconds to respond
setTimeout(function() {
worker.send(args[0].toString());
}, parseInt(args[0].toString())*10);
});
setTimeout((function() {
var value;
console.log('WORKER STARTING UP');
// the worker starts the communication, indicating it's ready
// rather than the queue just blindly sending work
worker.send('ready'); // sending the first message, which we catch above
}), 500); // In my experience, half a second is more than enough, YMMV
... as you can see, the pattern is:
Worker indicates readiness
Queue sends available work
Worker completes work and sends back
Queue receives completed work and sends back more work
GOTO 3
I'm playing with the idea of retrofitting spray-can with the simplistic node.js/http API.
Node allows to send a fixed length response (i.e. not chunked) in separate TCP packets. In the example below, the message is sent in eight packets. The first contains the HTTP head and the first chunk of 'x's, and the following packets contain 1024 'x's each (verified with Wireshark on OS X).
Is it possible to achieve the same with spray-can/http?
var http = require('http'),
server = http.createServer(function (request, response) {
var count = 8, size = 1024;
response.writeHead(200, 'honky-dory', {
'Content-Type': 'text/plain',
'Content-Length': count * size
});
(function send () {
setTimeout(function () {
response.write(Array(size + 1).join('x'));
count -= 1;
if(count > 0) send();
else response.end();
}, 10);
}());
});
server.listen(8080);
From what I saw so far, HttpResponse is rather 'atomic'.
UPDATE:
Based on the answers of jrudolph and Yuli Matsai, I came up with a listener which does more or less the same as the above node server. Actually it does a little more, as it doesn't allow flooding the network, because of the acknowledgements involved.
class Listener extends Actor {
def receive = {
case Connected (_, _) => sender ! Register (self)
case _: HttpRequest =>
val hs = HttpHeaders.`Content-Length` (8192) :: Nil
sender ! ChunkedResponseStart (HttpResponse (headers = hs)).withAck (8)
case 0 =>
sender ! ChunkedMessageEnd
case remaining: Int =>
sender ! MessageChunk (body = "x" * 1024).withAck (remaining - 1)
}
}
In order to be allowed to explicitly supply a Content-Length, the following entry must be enabled in application.conf:
spray.can.server {
chunkless-streaming = "on"
}
please check this one http://spray.io/documentation/1.1-SNAPSHOT/spray-routing/advanced-topics/response-streaming/ and there is good examples how to work with chenked data http://spray.io/documentation/1.1-SNAPSHOT/spray-routing/examples/#complete-examples
I'm using node.js as a server between pairs of clients, to handle my online game.
Clients send short messages between hem [one message should not exceed 200bytes].
Currently I expect single client to send [on average] 1 message per second [keeping in mind it can be 5 seconds of nothing and 5 messages one after another].
I've downloaded a sample server using 'net' module and rewritten it to handle the messages the way I need them to be handled.
Basically, for every connected socket, it creates a Buffer with size of 1024*8.
Currently I'm testing my game with some bots, which simply connect, wait 3 seconds and disconnect. They only send 1 message. Nothing else happening.
function sendMessage(socket, message) {
socket.write(message);
}
server.on('connection', function(socket) {
socket.setNoDelay(true);
socket.connection_id = require('crypto').createHash('sha1').update( 'krystian' + Date.now() + Math.random() ).digest('hex') ; // unique sha1 hash generation
socket.channel = '';
socket.matchInProgress = false
socket.resultAnnounced = false;
socket.buffer = new Buffer(cfg.buffer_size);
socket.buffer.len = 0; // due to Buffer's nature we have to keep track of buffer contents ourself
_log('New client: ' + socket.remoteAddress +':'+ socket.remotePort);
socket.on('data', function(data_raw) { // data_raw is an instance of Buffer as well
if (data_raw.length > (cfg.buffer_size - socket.buffer.len)) {
_log("Message doesn't fit the buffer. Adjust the buffer size in configuration");
socket.buffer.len = 0; // trimming buffer
return false;
}
socket.buffer.len += data_raw.copy(socket.buffer, socket.buffer.len); // keeping track of how much data we have in buffer
var str, start, end
, conn_id = socket.connection_id;
str = socket.buffer.slice(0,socket.buffer.len).toString();
if ( (start = str.indexOf("<somthing>")) != -1 && (end = str.indexOf("</something>")) != -1) {
try {
if (!<some check to see if the message format is right>) {
sendMessage(socket, "<error message to the client>");
return;
}
<storing info on the socket>
} catch(err) {
sendMessage(socket, "<error message to the client>");
return;
}
socket.channel = <channel>;
str = str.substr(end + 11);
socket.buffer.len = socket.buffer.write(str, 0);
sockets[socket.channel] = sockets[socket.channel] || {}; // hashmap of sockets subscribed to the same channel
sockets[socket.channel][conn_id] = socket;
waiting[socket.channel] = waiting[socket.channel] || {};
waiting[socket.channel][conn_id] = socket;
sendMessage(socket, "<info message to the client>");
for (var prop in waiting[socket.channel]) {
if (waiting[socket.channel].hasOwnProperty(prop) && waiting[socket.channel][prop].connection_id != socket.connection_id) {
<here I'll try to advertise this client among other clients>
sendMessage(waiting[socket.channel][prop], "<info to other clients about new client>");
}
}
}
var time_to_exit = true;
do{ // this is for a case when several messages arrived in buffer
if ( (start = str.indexOf("<some other format>")) != -1 && (end = str.indexOf("</some other format>")) != -1 ) {
var json = str.substr( start+19, end-(start+19) );
var jsono;
try {
jsono = JSON.parse(json);
} catch(err) {
sendMessage(socket, "<parse error>");
return;
}
if (<message indicates two clients are going to play together>) {
if (waiting[socket.channel][jsono.other_client_id] && waiting[socket.channel][socket.connection_id]) {
delete waiting[socket.channel][jsono.other_client_id];
delete waiting[socket.channel][socket.connection_id];
var opponentSocket = sockets[socket.channel][jsono.other_client_id];
sendMessage(opponentSocket, "<start game with the other socket>");
opponentSocket.opponentConnectionId = socket.connection_id;
sendMessage(socket, "<start game with the other socket>");
socket.opponentConnectionId = jsono.other_client_id;
}
} else if (<check if clients play together>) {
var opponentSocket = sockets[socket.channel][socket.opponentConnectionId];
if (<some generic action between clients, just pass the message>) {
sendMessage(sockets[socket.channel][socket.opponentConnectionId], json);
} else if (<match is over>) {
if (<match still in progress>) {
<send some messages indicating who won, who lost>
} else {
<log an error>
}
delete sockets[socket.channel][opponentSocket.connection_id];
delete sockets[socket.channel][socket.connection_id];
}
}
str = str.substr(end + 20); // cut the message and remove the precedant part of the buffer since it can't be processed
socket.buffer.len = socket.buffer.write(str, 0);
time_to_exit = false;
} else { time_to_exit = true; } // if no json data found in buffer - then it is time to exit this loop
} while ( !time_to_exit );
}); // end of socket.on 'data'
socket.on('close', function(){ // we need to cut out closed socket from array of client socket connections
if (!socket.channel || !sockets[socket.channel]) return;
if (waiting[socket.channel] && waiting[socket.channel][socket.connection_id]) {
delete waiting[socket.channel][socket.connection_id];
}
var opponentSocket = sockets[socket.channel][socket.opponentConnectionId];
if (opponentSocket) {
sendMessage(opponentSocket, "<the other client has disconnected>");
delete sockets[socket.channel][socket.opponentConnectionId];
}
delete sockets[socket.channel][socket.connection_id];
_log(socket.connection_id + " has been disconnected from channel " + socket.channel);
}); // end of socket.on 'close'
}); // end of server.on 'connection'
server.on('listening', function(){ console.log('Listening on ' + server.address().address +':'+ server.address().port); });
server.listen(cfg.port);
I've pasted the above code [very stripped version of the original] to give you and idea about how simple the server is.
I've got an array of sockets, who joined the game and array of sockets on the waiting list, waiting for another client to play with.
Nothing else is going on.
Still the script is memory hungry - 5 hours of connecting and disconnecting gave me this:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
31461 ec2-user 20 0 995m 91m 7188 S 0.7 15.4 1:29.07 node
I think this is way too much.
I'm using nodetime.com free service at the moment to monitor the script, but none of the metrics would suggest the script gained so much memory (it starts with just 10-12MB).
I believe this is due to the buffers, and because they allocate too much memory.
I'm only wondering, if my assumptions regarding buffer size are correct.
Should I adjust the buffer to reflect the amount of data I expect from the client?
If I expect the client to send 5 messages with a very short time between them, 200 bytes max each, should I assume that 1024*3 would be enough?
Or should I adjust buffer size according to the message size I expect, so if I'm sure the message will never go above 300 bytes, I should be fine with buffer size of 512?
Thanks,
Krystian
EDIT:
Node version:
$ node -v
v0.10.5
$ npm -v
1.2.19
EDIT2:
I've tested the script with 400 connections connecting and disconnecting and memory usage dropped significantly to around 60MB. After changing the test setup back to 4 connections it went up again.
The kernel has a socket receive buffer which is at least 8k., which takes care of multiple incoming messages on the socket. You don't need to buffer messages you've already read, so your application buffer doesn't need to be any bigger than the largest expected message.