Trying to write a TCP client in Node v0.10.15 and I am having a little trouble getting data back from the server. I know that the server is working properly because I have 3-4 different clients written in different languages communicating with it.
Below is a snippet of a larger piece of code but this should get the point across.
The problem is: I'm expecting 2 packets coming back after writing to the socket (this part is not included in this example). I'm only seeing the "data" event being fired once. Is there something that I need to do to get node to resume reading from the Tcp stream? I can confirm that the server is sending 2 packets(The length and then the actual data) Any help would be appreciated.
var dc = require('./DataContracts.js');
var net = require('net');
require('buffertools').extend();
var client = net.Socket();
var isConnected = false;
var serverHost = '10.2.2.21';
var dataCallback;
var receivedBuffer = new Array();
function InitComm(buffer) {
if (!isConnected) {
client.connect(4987, serverHost, function() {
client.on('data', function(data) {
console.log('Received server packet...');
var buf = new Buffer(data);
receivedBuffer.push(buf);
client.resume();
});
client.on('end', function() {
if (receivedBuffer.length > 1) {
if (dataCallback !== undefined)
dataCallback(receivedBuffer);
}
});
client.on('close', function() {
//clean up
});
client.on('error', function(err) {
console.log('Error!: ' + err);
});
Communicate(buffer);
});
} else {
Communicate(buffer);
}
}
Turns out that node was combining both of the packets together. I must be missing a Carriage return on the first packet
Related
I trying to create a socket server over the net-module in node.js. There should be different clients connect to it and be able to send/receive commands which will be evaluated on the server in the future.
And also clients should be able to ask for files which the server will send to them.
So, for example, one client is sending the command "file picture.jpg" and then the server will send the content of file picture.jpg through the socket and the client will download it to download.jpg.
This works fine for files <=2kB but not above. It seems like the client gets the files split up in different packages. Which would be fine if I only send files through sockets (So I would write every "package" into the file), but the problem is if I send big files which takes minutes and in that time the server sends other commands (which cant wait for the file transfer to complete).
Here is my simplified code:
server.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
// All connected socket clients will be in this list:
var client = [];
const server = net.createServer(function(sock) {
// We have a connection - a socket object is assigned to the connection automatically
console.log('Client connected: ' + sock.remoteAddress + ':' + sock.remotePort);
// Put this new client in the list
client.push(sock);
sock.on('data', function(data) {
const dataStr = data.toString();
const cmd = dataStr.split(' ', 1)[0];
const value = dataStr.slice(cmd.length).trim();
if (cmd == "file") {
fs.readFile(value, 'utf8', function(err, contents) {
if (err) {
if (err.code == 'ENOENT') {
sock.write('File not exist: '+value); // Send error to client
} else {
console.error(err);
}
} else {
sock.write('file '+contents); // Send file to client
}
});
console.log("Sending file to client.");
} else if (cmd == "quit") {
sock.write('Bye!');
sock.destroy();
} else {
sock.write('Unknown command: "' + dataStr + '"');
}
});
sock.on('error', (e) => {
if(e.code == "ECONNRESET") {
console.log("Got ECONNRESET, continue!");
} else {
console.log(e);
}
});
sock.on('close', function(data) {
var clientId = client.indexOf(sock);
client.splice(clientId, 1);
console.log('Client closed connection: ' + sock.remoteAddress +':'+ sock.remotePort);
});
});
server.on('error', function (e) {
if (e.code == 'EADDRINUSE') {
console.log('Creating server failed: Address in use ' + host +':'+ port);
}
});
server.listen(PORT, HOST, () => {
console.log('Server listening on ' + HOST +':'+ PORT);
});
client.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
var client = new net.Socket();
process.stdin.setEncoding('utf8');
var stdin = process.openStdin();
client.on('error', function (e) {
console.log("ERROR: "+e.code);
process.exit(1);
});
client.connect(PORT, HOST, function() {
console.log('Connected to server: ' + HOST + ':' + PORT);
process.stdout.write('> ');
});
stdin.addListener("data", function(d) {
var inp = d.toString().trim(); // clean input functions here
if (inp === "exit") {
//client.write("goodbye");
client.destroy();
stdin.destroy();
} else {
client.write(inp);
}
});
client.on('data', function(data) {
if (data.toString().substr(0, 5) === 'file ') { // If receiving file from server, data starts with "file "
const filename = "downloaded.jpg";
fs.writeFile(filename, data.toString().substr(5), function(err) {
if(err) {
return console.log(err);
}
console.log("Saved "+filename+".");
process.stdout.write('> ');
});
} else { // Unhandeld commands will be printed on console:
console.log('DATA: ' + data);
process.stdout.write('> ');
}
});
client.on('close', function() {
console.log('Connection closed');
process.exit();
});
So any suggestions how I solve this the best? Also, can I simply expand the buffersize somehow in net.sockets to like 32MB?
Since TCP is emulating a stream, you don't want to rely on anything about how the stream is broken into separate data events. The data given to one callback could be the first half of something or 2 things.
Instead, you want to emulate your datagram protocol on top of this reliable stream by appending stream contents to the end of a buffer and removing complete messages from the front for processing.
For example, this simple server from the docs, demonstrates a minimal valid TCP server:
const net = require('net');
const server = net.createServer((socket) => {
let name = '';
socket.setEncoding('utf8');
socket.on('data', (chunk) => name += chunk);
socket.on('end', () => socket.end(`hello ${name}`));
});
server.listen(8000);
It assembles a buffer with no assumption about the number of data call(s) in its simple case, the buffer is a single message to use at the end event.
To process messages before the end of the connection, you also want to examine the front of the buffer at the end of every data event to look if some messages are complete and ready to process. This separation of complete messages needs to be part of your protocol.
While message separation can be done by length indicators or reserved sequences, reserved sequences require encoding files (to avoid accidentally seeing them in data) and scanning data continuously to find them. This makes length indicators preferable for dealing with the file data.
So for example, the file [data] response first becomes file [#####] [data] where ##### tells you how much data to keep assembling on the buffer before a data callback will remove this entire message from the front of the buffer for processing as a fileSave().
Then, to handle more granular interactivity, simply break up these operations into separate smaller interactions, for example replace file [wholefilecount] [data] responses with filechunk [0-maxchunk] [data] responses that require a filenext command to continue and send a fileend to tell the client the file is done.
I am making use of "socket.io-client" and "socket.io stream" to make a request and then stream some data. I have the following code that handles this logic
Client Server Logic
router.get('/writeData', function(req, res) {
var io = req.app.get('socketio');
var nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
var nameNodeData = {};
async.waterfall([
checkForDataNodes,
readFileFromS3
], function(err, result) {
if (err !== null) {
res.json(err);
}else{
res.json("Finished Writing to DN's");
}
});
function checkForDataNodes(cb) {
nameNodeSocket.on('nameNodeData', function(data) {
nameNodeData = data;
console.log(nameNodeData);
cb(null, nameNodeData);
});
if (nameNodeData.numDataNodes === 0) {
cb("No datanodes found");
}
}
function readFileFromS3(nameNodeData, cb) {
for (var i in nameNodeData['blockToDataNodes']) {
var IP = nameNodeData['blockToDataNodes'][i]['ipValue'];
var dataNodeSocket = io.connect('http://'+ IP +":5000");
var ss = require("socket.io-stream");
var stream = ss.createStream();
var byteStartRange = nameNodeData['blockToDataNodes'][i]['byteStart'];
var byteStopRange = nameNodeData['blockToDataNodes'][i]['byteStop'];
paramsWithRange['Range'] = "bytes=" + byteStartRange.toString() + "-" + byteStopRange.toString();
//var file = require('fs').createWriteStream('testFile' + i + '.txt');
var getFileName = nameNodeData['blockToDataNodes'][i]['key'].split('/');
var fileData = {
'mainFile': paramsWithRange['Key'].split('/')[1],
'blockName': getFileName[1]
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
s3.getObject(paramsWithRange).createReadStream().pipe(stream);
//dataNodeSocket.disconnect();
}
cb(null);
}
});
Server Logic (that gets the data)
var dataNodeIO = require('socket.io')(server);
var ss = require("socket.io-stream");
dataNodeIO.on('connection', function(socket) {
console.log("Succesfully connected!");
ss(socket).on('sendData', function(stream, data) {
var IP = data['ipValue'];
var blockName = data['blockName'];
var mainFile = data['mainFile'];
dataNode.makeDir(mainFile);
dataNode.addToReport(mainFile, blockName);
stream.pipe(fs.createWriteStream(mainFile + '/' + blockName));
});
});
How can I properly disconnect the connections in function readFileFromS3. I have noticed using dataNodeSocket.disconnect() at the end does not work as I cannot verify the data was received on the 2nd server. But if I comment it out, I can see the data being streamed to the second server.
My objective is to close the connections in Client Server side
It appears that the main problem with closing the socket is that you weren't waiting for the stream to be done writing before trying to close the socket. So, because the writing is all asynchronous and finishes sometime later, you were trying to close the socket before the data had been written.
Also because you were putting asynchronous operations inside a for loop, you were also running all your operations in parallel which may not be exactly what you want as it makes error handling more difficult and server load more difficult.
Here's the code I would suggest that does the following:
Create a function streamFileFromS3() that streams a single file and returns a promise that will notify when it's done.
Use await in a for loop with that streamFileFromS3() to serialize the operations. You don't have to serialize them, but then you would have to change your error handling to figure out what to do if one errors while the others are already running and you'd have to be more careful about concurrency issues.
Use try/catch to catch any errors from streamFileFromS3().
Add error handling on the stream.
Change all occurrences of data['propertyName'] to data.propertyName. The only time you need to use brackets is if the property name contains a character that is not allowed in a Javascript identifier or if the property name is in a variable. Otherwise, the dot notation is preferred.
Add socket.io connection error handling logic for both socket.io connections.
Set returned status to 500 when there's an error processing the request
So, here's the code for that:
const ss = require("socket.io-stream");
router.get('/writeData', function(req, res) {
const io = req.app.get('socketio');
function streamFileFromS3(ip, data) {
return new Promise((resolve, reject) => {
const dataNodeSocket = io.connect(`http://${ip}:5000`);
dataNodeSocket.on('connect_error', reject);
dataNodeSocket.on('connect_timeout', () {
reject(new Error(`timeout connecting to http://${ip}:5000`));
});
dataNodeSocket.on('connection', () => {
// dataNodeSocket connected now
const stream = ss.createStream().on('error', reject);
paramsWithRange.Range = `bytes=${data.byteStart}-${data.byteStop}`;
const filename = data.key.split('/')[1];
const fileData = {
'mainFile': paramsWithRange.Key.split('/')[1],
'blockName': filename
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
// get S3 data and pipe it to the socket.io stream
s3.getObject(paramsWithRange).createReadStream().on('error', reject).pipe(stream);
stream.on('close', () => {
dataNodeSocket.disconnect();
resolve();
});
});
});
}
function connectError(msg) {
res.status(500).send(`Error connecting to ${NAMENODE_ADDRESS}`);
}
const nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
nameNodeSocket.on('connect_error', connectError).on('connect_timeout', connectError);
nameNodeSocket.on('nameNodeData', async (nameNodeData) => {
try {
for (let item of nameNodeData.blockToDataNodes) {
await streamFileFromS3(item.ipValue, item);
}
res.json("Finished Writing to DN's");
} catch(e) {
res.status(500).json(e);
}
});
});
Other notes:
I don't know what paramsWithRange is as it is not declared here and when you were doing everything in parallel, it was getting shared among all the connections which is asking for a concurrency issue. In my serialized implementation, it's probably safe to share it, but the way it is now bothers me as it's a concurrency issue waiting to happen.
I'm totally new to the whole nodeJS asynchronous-y callback-y programming so I need more like a guidance to understanding what I'm even doing. With that said, I have two files main.js and server.js
My main file looks like this:
var server=require('./server.js');
server();
function WhenUserClicksButton(){
server();
}
and my server file looks like this:
var net = require('net');
function server(){
net.createServer(function (socket) {
socket.write('\x16'); //SYN character
socket.on('data', function (data) {
//handle data from client
});
}).listen(33333);
}
First call of server(); starts the TCP server. Then function WhenUserClicksButton is called when user clicks button (duhh) in a GUI. But it attempts to start the server again so I get
Error: listen EADDRINUSE :::33333
I got why this is happening but I can't think of a solution for it. What I really need is:
Start the server and listen on 33333
When nothing is happening server and client just exchanges SYN and ACK characters every few seconds (I already have this part done, I just removed it from this example for clarity because it's not really topic of this question)
When user click button change socket.write('\x16'); to socket.write('something');
Then wait for server and client to exchange data and after everything is done return results back to main.js
As I said, I'm new to this and I believe my problem lies in not understanding fully of what I'm doing. Any help and explanations are welcome!
I think you're very near where you need to be. I would do something like this:
server.js
var net = require('net');
var netServer = null;
var netSocket = null;
function sendData(data) {
if (netServer && netSocket) {
console.log('Send data: sending: ', data);
netSocket.write(data);
}
}
function startServer(){
netServer = net.createServer(function (socket) {
netSocket = socket;
socket.write('\x16'); //SYN character
socket.on('data', function (data) {
console.log('Server: data from client: ', data);
if (data.length === 1 && data[0] === 0x16) {
// log and ignore SYN chars..
console.log('SYN received from client');
} else if (newDataCallback) {
newDataCallback(data);
};
});
});
console.log('Server listening on 33333..');
netServer.listen(33333);
}
var newDataCallback = null;
function setNewDataCallback(callback) {
newDataCallback = callback;
}
module.exports = {
sendData: sendData,
startServer: startServer,
setNewDataCallback: setNewDataCallback
};
main.js
var server = require('./server');
function newDataCallback(data) {
console.log('newDataCallback: New data from server: ', data);
}
server.setNewDataCallback(newDataCallback);
server.startServer();
function wheneverUserClicksButton() {
server.sendData('something');
}
testClient.js
var clientSocket = net.createConnection(33333, "127.0.0.1");
clientSocket.on('data', (someData) => {
console.log('Data received', someData);
});
clientSocket.on('connect', () => {
console.log('Client Socket connected ');
clientSocket.write('Hello from client');
});
i want to readout data from 2 diffrent IP adresses. But i always get "EALREADY" Error.
Which means that the Socket is "already in use", although i'm destroying the socket after i received the data.
If i remove the for loop and only readout 1 adress, it works just fine...
If i'm adding timeouts they won't fire... and i don't know why either.
var socket = new net.Socket();
var test = [];
test[0] = '10.5.0.5';
test[1] = '10.5.0.11';
for(var i=0;i<test.length;i++) {
socket.connect('2711', test[i], function() {
socket.write('d\n');
console.log("1");
console.log(test[i]);
socket.on('data', function(data) {
data = data.toString();
console.log(data);
socket.destroy();
console.log("2");
});
});
}
Help highly appreciated.
Thanks in advance.
The "For loop" is synchronous but ".connect" and ".on('data'" are asynchronous, therefore the for loop finished before the first connection happens. The second connection attempt happens when first one is still in progress, that is why you get connection error.
You should try a recursive function, advancing step by step. For example I create a function that receives an array, and iterate the first element of array, connect, and on data close it and call again the function if array still has more items.
var socket = new net.Socket();
var test = [];
test[0] = '10.5.0.5';
test[1] = '10.5.0.11';
function testIP(test){
socket.connect('2711', test.shift(), function() {
socket.write('d\n');
console.log("1");
socket.on('data', function(data) {
data = data.toString();
console.log(data);
socket.destroy();
console.log("2");
if( test.length > 0 ){
testIP(test);
}
});
}
testIP(test);
Bonus: There are libraries in Node.js to work with asynchronous tasks
I have the following situation and just wanted to check if I am doing it right. I have a couple of devices at my customers (RS232). Now I have a RS232-WIFI dongle connected with it, so data out of the device is sent over to my server (it outputs a datastring, example: {12,1,etc). On my server I have a NodeJS script running, that opens up a port and fetches all data coming in.
var net = require('net');
var host = '1.1.1.1';
var servers = [];
var ports = [20000, 20001, 20002, 20003, 20004];
// Create servers
ports.forEach(function (port) {
var s = net.createServer(function (sock) {
// We have a connection - a socket object is assigned to the connection automatically
console.log('CONNECTED (' + sock.localPort + '): ' + sock.remoteAddress + ':' + sock.remotePort);
// Add a 'data' event handler to this instance of socket
sock.on('data', function (data) {
// post data to a server so it can be saved and stuff
postData(data.toString(), sock);
// close connection
sock.end();
});
sock.on('error', function (error) {
console.log('******* ERROR ' + error + ' *******');
// close connection
sock.end();
});
});
s.listen(port, host, function () {
console.log('Server listening on ' + host + ':' + s.address().port);
});
servers.push(s);
});
Okay, so this works pretty good. But I am wondering, sometimes not all of the data is posted at once, sometimes I get {12, and after a second I get the rest (or even more times is needed). What can I do to optimize this script further? Do I need to call sock.end(); after receiving data? Does this hurt network performance for me or my customers?
If you guys need more info let me know.
That depends on the protocol of your devices, if the devices use each connection for a chunk of data, you can write the program like so: (Do not close the socket on data event)
....
// socket will close and destroy automatically after the device close the connection
var s = net.createServer(function (sock) {
sock.setEncoding('utf8');
var body = "";
sock.on('data', function (data) {
body = body + data;
});
sock.on('end', function() {
console.log(data);
postData(data);
});
// TODO error handling here
});
....
Note: Socket is not guaranteed to give you all data at once, you should listen data event then concat all chunks before using.
If your devices don't close socket, you will not receive on('end'), then the code should be like this:
....
var s = net.createServer(function (sock) {
sock.setEncoding('utf8');
// var body = "";
sock.on('data', function (data) {
// body = body + data;
postData(data);
});
sock.on('end', function() {
console.log('end');
});
// TODO error handling here
});
....