Sending files through net.socket - node.js

Question

I trying to create a socket server over the net-module in node.js. There should be different clients connect to it and be able to send/receive commands which will be evaluated on the server in the future.
And also clients should be able to ask for files which the server will send to them.
So, for example, one client is sending the command "file picture.jpg" and then the server will send the content of file picture.jpg through the socket and the client will download it to download.jpg.
This works fine for files <=2kB but not above. It seems like the client gets the files split up in different packages. Which would be fine if I only send files through sockets (So I would write every "package" into the file), but the problem is if I send big files which takes minutes and in that time the server sends other commands (which cant wait for the file transfer to complete).
Here is my simplified code:
server.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
// All connected socket clients will be in this list:
var client = [];
const server = net.createServer(function(sock) {
// We have a connection - a socket object is assigned to the connection automatically
console.log('Client connected: ' + sock.remoteAddress + ':' + sock.remotePort);
// Put this new client in the list
client.push(sock);
sock.on('data', function(data) {
const dataStr = data.toString();
const cmd = dataStr.split(' ', 1)[0];
const value = dataStr.slice(cmd.length).trim();
if (cmd == "file") {
fs.readFile(value, 'utf8', function(err, contents) {
if (err) {
if (err.code == 'ENOENT') {
sock.write('File not exist: '+value); // Send error to client
} else {
console.error(err);
}
} else {
sock.write('file '+contents); // Send file to client
}
});
console.log("Sending file to client.");
} else if (cmd == "quit") {
sock.write('Bye!');
sock.destroy();
} else {
sock.write('Unknown command: "' + dataStr + '"');
}
});
sock.on('error', (e) => {
if(e.code == "ECONNRESET") {
console.log("Got ECONNRESET, continue!");
} else {
console.log(e);
}
});
sock.on('close', function(data) {
var clientId = client.indexOf(sock);
client.splice(clientId, 1);
console.log('Client closed connection: ' + sock.remoteAddress +':'+ sock.remotePort);
});
});
server.on('error', function (e) {
if (e.code == 'EADDRINUSE') {
console.log('Creating server failed: Address in use ' + host +':'+ port);
}
});
server.listen(PORT, HOST, () => {
console.log('Server listening on ' + HOST +':'+ PORT);
});
client.js
var net = require('net');
var fs = require('fs');
var HOST = '127.0.0.1';
var PORT = 6969;
var client = new net.Socket();
process.stdin.setEncoding('utf8');
var stdin = process.openStdin();
client.on('error', function (e) {
console.log("ERROR: "+e.code);
process.exit(1);
});
client.connect(PORT, HOST, function() {
console.log('Connected to server: ' + HOST + ':' + PORT);
process.stdout.write('> ');
});
stdin.addListener("data", function(d) {
var inp = d.toString().trim(); // clean input functions here
if (inp === "exit") {
//client.write("goodbye");
client.destroy();
stdin.destroy();
} else {
client.write(inp);
}
});
client.on('data', function(data) {
if (data.toString().substr(0, 5) === 'file ') { // If receiving file from server, data starts with "file "
const filename = "downloaded.jpg";
fs.writeFile(filename, data.toString().substr(5), function(err) {
if(err) {
return console.log(err);
}
console.log("Saved "+filename+".");
process.stdout.write('> ');
});
} else { // Unhandeld commands will be printed on console:
console.log('DATA: ' + data);
process.stdout.write('> ');
}
});
client.on('close', function() {
console.log('Connection closed');
process.exit();
});
So any suggestions how I solve this the best? Also, can I simply expand the buffersize somehow in net.sockets to like 32MB?

Answer 1

Since TCP is emulating a stream, you don't want to rely on anything about how the stream is broken into separate data events. The data given to one callback could be the first half of something or 2 things.
Instead, you want to emulate your datagram protocol on top of this reliable stream by appending stream contents to the end of a buffer and removing complete messages from the front for processing.
For example, this simple server from the docs, demonstrates a minimal valid TCP server:
const net = require('net');
const server = net.createServer((socket) => {
let name = '';
socket.setEncoding('utf8');
socket.on('data', (chunk) => name += chunk);
socket.on('end', () => socket.end(`hello ${name}`));
});
server.listen(8000);
It assembles a buffer with no assumption about the number of data call(s) in its simple case, the buffer is a single message to use at the end event.
To process messages before the end of the connection, you also want to examine the front of the buffer at the end of every data event to look if some messages are complete and ready to process. This separation of complete messages needs to be part of your protocol.
While message separation can be done by length indicators or reserved sequences, reserved sequences require encoding files (to avoid accidentally seeing them in data) and scanning data continuously to find them. This makes length indicators preferable for dealing with the file data.
So for example, the file [data] response first becomes file [#####] [data] where ##### tells you how much data to keep assembling on the buffer before a data callback will remove this entire message from the front of the buffer for processing as a fileSave().
Then, to handle more granular interactivity, simply break up these operations into separate smaller interactions, for example replace file [wholefilecount] [data] responses with filechunk [0-maxchunk] [data] responses that require a filenext command to continue and send a fileend to tell the client the file is done.