Nodejs Sending multiple files - node.js

I'm trying to send multiple files to a used and have them displayed, for now I'm testing with simple txt files.
Since piping more than 10 files at a time causes warnings/errors I have tried to only pipe 10 at a time. The function gets called the right amount of times but, for some reason, only sends part of the data ( always seems to stop at 2.3mb ). Any reason why?
app.get('/file', function(req, res) {
console.time('leggo');
var lowerbound = req.param('lowerbound'); // 3 vars used for the file.
var upperbound = req.param('upperbound');
var filename = req.param('filename');
res.writeHead(200, {
'Content-type' : 'text/plain',
'Content-disposition' : 'inline'
});
var readst = fs.createReadStream(base+ lowerbound + last); //first file.
var count = 0; // number of files that are being sent atm.
var add = function(i) { // add i'th file to the pipe
console.log('whatevs');
++i;
readst.push(base + i + last);
};
var started = 0;
function sendem() { //send files.
++count;
++started;
var mycallback = function () { //callback when doit finished
console.log(count + ' ' + started + '\n');
count -= 1;
if (started == upperbound)
return;
if (count < 10) sendem();
};
function doit(mycallback) { //send a file then call callback
add(started);
mycallback();
};
doit(mycallback); //start sending.
}
sendem(); // start the "main" sending function.
readst.pipe(res); // pipe everything, also tried using this earlier, same effect.
readst.on('end', function(err,data) { // "everything" was sent.
console.log('done');
console.timeEnd('leggo');
});
});

Related

control doesnot enter in req.on in node.js

I am using req.on() inmy project but it doesnot enter in the function I don't know why because it was working till last night.
exports._login= function(req,res){
console.log("1111");
// Post data may be sent in chunks so need to build it up
var body ="";
console.log("here")
// From here without entering in funtion it goes to last line (outsidefunction)
req.on("data", (data) =>
{
console.log("in req.on function")
body += data;
console.log("abc"+body);
var vars = body.split("&");
console.log("vars "+vars);
var value =[];
for (var t = 0; t < vars.length; t++)
{
var pair = vars[t].split("=");
var key = decodeURIComponent(pair[0]);
var val = decodeURIComponent(pair[1]);
value.push( decodeURIComponent(pair[1]));
console.log(key + ":" + val);
}
console.log(value[0]+"******"+value[1]);
username=value[0];
userPassword=value[1]
console.log("out function")
username=req.query.usernamePost;
userPassword=req.query.passwordPost;
console.log(username+"hello");
});
console.log("last");
}
I cant figure out what is the problem kindly help

createWriteStream to memory buffer and push to disk later

I'm really struggling with connect-busboy and stream writing. The problem is I want to do some file validation but I can't do that without processing the data first and it seems like I have to write this to a file while I process it.
When the data comes in and I come across errors like file size or file types restrictions, I reject the request but a portion of the file is already written to disk. I really want to omit writing until the very end or perhaps stage it into a memory buffer.
This code will reject on invalid file types and size limits, but it always first creates the write stream and pipes the file into it. What I really want to do it move the "file.pipe(fstream);" into the file.on('end',... callback but that of course does not work because it has to be written as it comes in. So it seems like I need to first write it to a staging buffer, then on end flush it to a real file. Alternatively I could write to a temporary file, and on error delete that file, but that seems far more hacky and fragile. What is the best practice for this seemingly common problem?
saveFiles: function(request, options) {
return new Promise(function(resolve, reject) {
var fstream;
var totalData = 0;
request.pipe(request.busboy);
var fileIds = [];
var assets = {'assets': {}};
request.busboy.on('file', function(fieldname, file, originalName, encoding, contype) {
file.on('limit', function() {
var limitBytes = request.busboy.opts.limits.fileSize;
var limitMB = Math.round((limitBytes / 1048576) * 100) / 100;
return reject({errorMsg: 'Files exceeded the size limit of ' + limitMB + 'MB'});
});
file.on('data', function(data) {
if (totalData === 0) {
var type = fileType(data);
if(type) {
if (request.busboy.opts.limits.validFileTypes.indexOf(type.ext) === -1) {
return reject({errorMsg: 'Invalid file type "' + type.ext + '"'});
}
}
else {
return reject({errorMsg: 'Invalid file type'});
}
}
totalData += data.length;
});
var fileId = uuid.v1(); // used for file name on server
var parts = originalName.split('.');
var extension = parts[parts.length - 1].toLowerCase();
var fileName = fileId + '.' + extension;
fstream = fs.createWriteStream(__assets + fileName);
file.pipe(fstream);
file.on('end', function() {
assets['assets'][fileId] = {
fileName: fileId + '.' + extension,
originalName: originalName
};
});
});
request.busboy.on('finish', function(){
resolve(assets);
});
});
}

node.js server not always clearing data from previous run

I'm brand new to node.js (or javascript in general) and not sure what's going on here. Sometimes when I run my node.js server, right when it starts up it begins printing data from the last time it ran. Is there something that could be keeping everything from resetting on a new run? I have arduino XBee nodes connecting to this server. Here's my node.js code:
var SerialPort = require("serialport");
var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var ON_DEATH = require('death');
// Set up serial port
var portName = process.argv[2],
portConfig = {
baudRate: 9600,
parser: SerialPort.parsers.readline("\n")
};
var sp = new SerialPort(portName, portConfig); // got rid of deprecation issue with sp4
// Respond with file when a GET request is made to the homepage: app.METHOD(PATH, HANDLER) '/'=homepage
app.get('/', function(req, res){
res.sendfile('index.html');
});
// Listen on the connection event for incoming sockets, and log it to the console.
io.on('connection', function(socket){
console.log('a user connected');
socket.on('disconnect', function(){
});
socket.on('chat message', function(msg){
io.emit('chat message', msg);
sp.write(msg + "\n");
});
});
// Make the http server listen on port 3000
http.listen(3000, function(){
console.log('listening on *:3000');
});
/////////////////////////////////////////////////////////////////////////////////////////////////
sp.on("open", function (err) {
if (err)
return console.log('Error opening port: ', err.message);
console.log('open');
// INFO VARIABLES
//var nodeCount = 0;
var pendingNodes = [];
var connectedNodes = [];
var buffer0;
var nodeInfo;
// Cleanup when termination signals are sent to process
ON_DEATH(function(signal, err) {
var death_msg = "Q";
sp.write(death_msg);
sp.close();
// zero out all variables
pendingNodes = [];
connectedNodes = [];
buffer0 = [];
nodeInfo = [];
console.log("\n\nSending reset signal to nodes.\n\n")
sp.flush(function(err,results){});
process.exit();
})
// listen for data, grab time, delimit
sp.on('data', function(data) {
var time = new Date();
buffer0 = data.split('\n'); // array of data till newline
// Split again into either "#ID" into [#ID] (if new broadcast), or "#ID Data" into [#ID, Data] (preconnected node)
nodeInfo = buffer0[0].split(' ');
//console.log(" nodeInfo: " + nodeInfo);
// DEBUGGING PRINT -- DELETE LATER
//console.log(" pendingNodes: " + pendingNodes + " connectedNodes: " + connectedNodes);
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Receiving ID or [ID, Data] -- Testing all possible cases
// if first char
if (nodeInfo[0][0] == "#") { // could check up here for && no temp data too
// Brand new node (ID not in pendingNodes or connectedNodes)
if ((pendingNodes.indexOf(nodeInfo[0]) == -1) && (connectedNodes.indexOf(nodeInfo[0]) == -1)) {
console.log("Brand new node: " + nodeInfo[0])
pendingNodes.push(nodeInfo[0]);
}
// Pending node (ID in pendingNodes, but not in connectedNodes)
else if ((pendingNodes.indexOf(nodeInfo[0]) != -1) && ((connectedNodes.indexOf(nodeInfo[0]) == -1))) {
console.log(" Pending node: " + nodeInfo[0])
// send back ID to confirm handshake
sp.write(nodeInfo[0]);
// Remove from pendingNodes
var index = pendingNodes.indexOf(nodeInfo[0]);
if (index > -1) {
pendingNodes.splice(index, 1);
}
// Add to connectedNodes
//var
connectedNodes.push(nodeInfo[0]);
}
// Connected node (ID in connectedNodes, but not in pendingNodes)
else if ((pendingNodes.indexOf(nodeInfo[0]) == -1) && (connectedNodes.indexOf(nodeInfo[0]) != -1)) {
console.log(" Connected node: " + nodeInfo[0] + " " + nodeInfo[1]);
// Disconnect nodes with undefined temperatures to force restart
if (typeof nodeInfo[1] == 'undefined') {
console.log("Undefined data for node: " + nodeInfo[0] + ". Dropping connection to this node.");
var index = connectedNodes.indexOf(nodeInfo[0]);
if (index > -1) {
connectedNodes.splice(index, 1);
}
}
//var t = time.getTime();
// FIRST: fix bug -- sometimes temp showing as undefined
// Then:
// Update data properly
// Average data
// add 3rd element to nodes for time stamp, check that all timestamps are recent periodically
// Detect drop-offs and remove nodes
}
// Error (ID in both pendingNodes and connectedNodes [should not happen])
else {
console.log("Error: node exists in both pending and connected.")
}
}
else if (nodeInfo[0][0] != "#") {
// FOR DEBUGGING PURPOSES, DELETE LATER
console.log("Error 2: incoming data does not start with '#'.")
}
/* // placeholders for functionality to add
var CalcAverage = function() {
console.log("CalcAverage: * every 1 second(s) *");
}
setInterval(function(){ CalcAverage() },2000);
var CheckNodes = function() {
console.log("CheckNodes: * every 6 second(s) *");
}
setInterval(function(){ CheckNodes() },6000);
*/
}); // end data
}); // end open
And here's an example of old node data showing up somehow:
Some possibilities:
Old (or new data that you didn't know it sent) data from clients has been buffered somehow on the serial port, or socket.io auto-reconnects on the client browsers and starts sending data even though you didn't interact with the client browser web page.

RecordRTC upload video to node js server

I am using RecordRTC from recording webrtc meeting. After implementing recording, when I test this application if both client are on the same system then its working fine. When I test this application on different system it isn't working fine and meeting is not recorded.
Here this is my code from stop recording client side.
recordRTC.stopRecording(function (videoURL) {
console.log('recordRTC.stopRecording Function inside');
SelectedFile = recordRTC.getBlob();
$('#uploadForm').append('#attachmentFileId', recordRTC.getBlob());
StartUpload();
});
var FReader;
var Name = "Meeting" + "_" + Date.now() + ".webm";
function StartUpload()
{
FReader = new FileReader();
FReader.onload = function (evnt)
{
socket.emit('Upload', { 'Name': Name, Data: evnt.target.result });
}
socket.emit('Start', { 'Name': Name, 'Size': SelectedFile.size });
}
socket.on('MoreData', function (data)
{
var Place = data['Place'] * 524288; //The Next Blocks Starting Position
var NewFile; //The Variable that will hold the new Block of Data
if (SelectedFile.webkitSlice)
NewFile = SelectedFile.webkitSlice(Place, Place + Math.min(524288, (SelectedFile.size - Place)));
else
NewFile = SelectedFile.slice(Place, Place + Math.min(524288, (SelectedFile.size - Place)));
FReader.readAsBinaryString(NewFile);
});
Server Side Code
I get this from here.
socket.on('Start', function (data) { //data contains the variables that we passed through in the html file
var Name = data['Name'];
Files[Name] = { //Create a new Entry in The Files Variable
FileSize : data['Size'],
Data : "",
Downloaded : 0
}
var Place = 0;
try{
var Stat = fs.statSync('Temp/' + Name);
if(Stat.isFile())
{
Files[Name]['Downloaded'] = Stat.size;
Place = Stat.size / 524288;
}
}
catch(er){} //It's a New File
fs.open("Temp/" + Name, 'a', 0755, function(err, fd){
if(err)
{
console.log(err);
}
else
{
Files[Name]['Handler'] = fd; //We store the file handler so we can write to it later
socket.emit('MoreData', { 'Place' : Place, Percent : 0 });
}
});
});
socket.on('Upload', function (data){
var Name = data['Name'];
Files[Name]['Downloaded'] += data['Data'].length;
Files[Name]['Data'] += data['Data'];
if(Files[Name]['Downloaded'] == Files[Name]['FileSize']) //If File is Fully Uploaded
{
fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', function(err, Writen){
var input = fs.createReadStream("Temp/" + Name);
var output = fs.createWriteStream("Video/" + Name);
//util.pump(readableStream, writableStream, [callback])
//Deprecated: Use readableStream.pipe(writableStream)
input.pipe(output);
input.on("end", function() {
console.log("end");
fs.unlink("Temp/" + Name, function ()
{ //This Deletes The Temporary File
console.log("unlink this file:",Name );
//socket.emit('Done', {'Image' : 'Video/' + Name + '.jpg'});
});
});
});
}
else if(Files[Name]['Data'].length > 10485760){ //If the Data Buffer reaches 10MB
fs.write(Files[Name]['Handler'], Files[Name]['Data'], null, 'Binary', function(err, Writen){
Files[Name]['Data'] = ""; //Reset The Buffer
var Place = Files[Name]['Downloaded'] / 524288;
var Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
socket.emit('MoreData', { 'Place' : Place, 'Percent' : Percent});
});
}
else
{
var Place = Files[Name]['Downloaded'] / 524288;
var Percent = (Files[Name]['Downloaded'] / Files[Name]['FileSize']) * 100;
socket.emit('MoreData', { 'Place' : Place, 'Percent' : Percent});
}
});
If both clients are on same machine/system its working fine, but if both clients are on different system then meeting is not recorded.

Node.js - MJPEG TCP stream to base64 images

Based on paparazzo.js lib, I'm trying to get base64 images from a MJPEG stream (streamed over TCP with GStreamer) in a Node.js server, and to send them to the clients via websockets.
I think I'm pretty close, but my images are corrupted. Here is the code I'm using :
var boundary = "----videoboundary";
var data = "";
var tcpServer = net.createServer(function (socket) {
socket.on('data', function(chunk) {
var boundaryIndex = chunk.toString().indexOf(boundary);
if (boundaryIndex !== -1) {
// Get the image's last piece of data :
data += chunk.toString().substring(0, boundaryIndex);
// Convert the data to a base64 image and broadcast it :
var image = new Buffer(data).toString('base64');
io.sockets.emit('image', image);
// Reset the data :
data = '';
// Get the remaining data (with new image's headers) :
var remaining = chunk.toString().substring(boundaryIndex);
// Remove the new image's headers and add the remaining data :
var contentTypeMatches = remaining.match(/Content-Type:\s+image\/jpeg\s+/);
var contentLengthMatches = remaining.match(/Content-Length:\s+(\d+)\s+/);
if(contentLengthMatches != null && contentLengthMatches.length > 1) {
var newImageBeginning = remaining.indexOf(contentLengthMatches[0]) + contentLengthMatches[0].length;
data += remaining.substring(newImageBeginning);
}
else if(contentTypeMatches != null) {
var newImageBeginning = remaining.indexOf(contentTypeMatches[0]) + contentTypeMatches[0].length;
data += remaining.substring(newImageBeginning);
}
else {
var newImageBeginning = boundaryIndex + boundary.length;
data += remaining.substring(newImageBeginning);
io.sockets.emit('error', { message: 'Could not find beginning of next image' });
}
}
else {
data += chunk;
}
});
});
Any idea ?
Thanks
chunk.toString() converts the binary Buffer to a utf8-encoded string (by default), so for binary image data that will probably cause you some problems.
Another option that might help simplify things for you is to use the dicer module. With that, your code may look like:
var Dicer = require('dicer');
var boundary = '----videoboundary';
var tcpServer = net.createServer(function(socket) {
var dice = new Dicer({ boundary: boundary });
dice.on('part', function(part) {
var frameEncoded = '';
part.setEncoding('base64');
part.on('header', function(header) {
// here you can verify content-type, content-length, or any other header
// values if you need to
}).on('data', function(data) {
frameEncoded += data;
}).on('end', function() {
io.sockets.emit('image', frameEncoded);
});
}).on('finish', function() {
console.log('End of parts');
});
socket.pipe(dice);
});

Resources