I have an implementation of a POST router in node.js with express. The body of the post is a list of files, and the idea is that these files should be zipped and a link to the resulting zip file should be returned. I'm trying to use 'express-session' to store the progress in terms of number of files zipped as well as the file name of the zip upon completion. Updating the number of files zipped (req.session.current) works fine, but for some reason I can't set the file name (req.session.zipFile) in the session when the fileStream closes. I'm suspecting that the req object is no longer valid by the time fileStream.on('close') is reached, but I'm not sure how to handle that. I can't access the session data without a request.
This is the complete router implementation:
const express = require('express');
const router = express.Router();
const fs = require('fs');
const ZIP_FILES_PATH = require('../constants');
router.post('/', function(req, res, next){
const json = JSON.parse(req.body.data);
const path = json["base"];
if (!path || path.length === 0){
res.render("error", {msg: "JSON missing 'base'"})
return;
}
const files = json["files"];
if (!files || files.length === 0){
res.statusCode = 400;
res.send("JSON missing 'files'");
return;
}
if (!fs.existsSync(path)) {
res.statusCode = 400;
res.send("Directory '" + path + "' does not exist");
return;
}
try{
var sessData = req.session;
sessData.total = files.length ;
sessData.current = 0;
sessData.zipFile = '';
zipFiles(path, files, req, res);
}catch(error){
res.statusCode = 500;
res.send("The files could not be zipped");
return;
}
res.render("zipping")
});
module.exports = router;
const zipFiles = (path, files, req, res) => {
const zipFile = require('crypto').createHash('md5').update(path).digest("hex") + "_" + new Date().getTime() + ".zip";
const archiver = require('archiver');
const fileStream = fs.createWriteStream(ZIP_FILES_PATH + zipFile);
const archive = archiver('zip', {
gzip: true,
zlib: { level: 9 }
});
archive.on('error', function(err) {
throw err;
});
fileStream.on('close', function() {
req.session.zipFile = "/download?file=" + zipFile; //Doesn't stick!
});
archive.pipe(fileStream);
files.map(file => zipSingleFile(path, file, req, archive));
archive.finalize();
}
const zipSingleFile = (path, file, req, archive) => {
fs.existsSync(path + "/" + file) ? archive.file(path + "/" + file, { name: file }) : null;
req.session.current = req.session.current + 1;
}
zipFiles() is asynchronous. That means it doesn't block and finishes some time later. So, in the flow of your request handler, you call
res.render("zipping")
before zipFiles() finishes. And, res.render() will send the response and close the output stream.
If you want to wait to call res.render() until you're done with the zipFiles() call, then you need to either call it from within zipFiles() when it's all done or you need to add a callback to zipFiles() so it can communicate back when it's done.
Also, is there any reason you can't set req.session.zipFile = "/download?file=" + zipFile; before you call res.render()?
Also, depending upon how you have your session configured (which you don't show us any code for), just doing this:
req.session.zipFile = "/download?file=" + zipFile;
may not be enough to actually save that value into the session. You may also have to call .save() on the session.
And, you are setting:
req.session.zipFile
at the end of your zipping process so any requests that arrive from that session before your zipping finishes won't see that session variable set either (this seems like a concurrency problem to me).
Related
I've been seeing performance issues on our application and I'm a bit unsure if uploading a file to S3 could block NodeJS.
I'm using express, formidable and aws-sdk.
Here's a middleware using formidable. This stores the file in req.file and continues to the next middleware that performs the upload to S3.
var formidable = require("formidable");
module.exports = function() {
return function(req, res, next) {
var form = new formidable.IncomingForm({
"keepExtensions": true,
"uploadDir": config.tempDir
});
form.parse(req, function(error, fields, files) {
if (error) {
return res.sendError("Error while parsing multipart-form " + error, 500);
}
req.files = files;
req.fields = fields;
next();
});
};
};
Here's the middleware that actually makes the request to S3 using AWS SDK
const AWS = require("aws-sdk");
const fs = require("fs");
const s3 = new AWS.S3(s3config.s3options);
const logger = require("logger");
function uploadFile(req, res, next) {
const requestId = getRequestIdFromRequestHeaders(req);
const file = options.file(req);
const contentType = options.contentType && options.contentType(req) || file.type;
const destinationPath = options.destinationPath(req);
req.s3uploaded = false;
logger.debug(requestId, "invoking uploadFile", contentType, destinationPath);
req.s3FilePath = "https://" + s3config.bucket + ".s3.amazonaws.com/" + destinationPath;
if (!options.writeConcern) {
logger.debug(requestId, "write concern not expected. Calling next");
next();
}
const stream = fs.createReadStream(file.path);
s3.upload({
"Bucket": s3config.bucket,
"ContentLength": file.size,
"Key": destinationPath,
"Body": stream,
"ContentType": contentType
}, s3config.s3options.uploadOptions, function(error) {
fs.unlink(file.path, error => {
if (error) {
logger.error(requestId, "Unable to remove file", file.path);
}
});
if (error) {
return next(error);
}
if (options.writeConcern) {
if (!req.s3uploaded) {
req.s3uploaded = true;
next();
}
}
}).on("httpUploadProgress", progress => {
logger.debug(requestId, "progress", progress.loaded, "of", progress.total);
if (progress.total !== undefined && progress.loaded === progress.total) {
logger.debug(requestId, "upload done, invoking next from httpUploadProgress");
if (!req.s3uploaded) {
req.s3uploaded = true;
next();
}
}
});
};
The documentation for the AWS SDK for JavaScript (v2) includes this statement on their Calling Services Asynchronously page (emphasis mine):
All requests made through the SDK are asynchronous. This is important to keep in mind when writing browser scripts. JavaScript running in a web browser typically has just a single execution thread. After making an asynchronous call to an AWS service, the browser script continues running and in the process can try to execute code that depends on that asynchronous result before it returns.
I have vue (axios) making a get call to an express route which triggers a child_process of ffmpeg in an infinite loop. ffmpeg streams one file over udp , on close it re calls itself and streams another file.
I'd like to be able to kill this process from a button on a web page, but can't seem to work it out.
This is my express route code
router.get('/test', function(req, res) {
const childProcess = require('child_process');
const fs = require('fs')
const path = require('path')
//Grabs a random index between 0 and length
function randomIndex(length) {
return Math.floor(Math.random() * (length));
}
function Stream () {
const FILE_SRC = '/path/to/file'
//Read the directory and get the files
const dirs = fs.readdirSync(FILE_SRC)
.map(file => {
return path.join(FILE_SRC, file);
});
const srcs_dup = [];
const hashCheck = {}; //used to check if the file was already added to srcs_dup
var numberOfFiles = dirs.length - 1; //OR whatever # you want
console.log(numberOfFiles)
//While we haven't got the number of files we want. Loop.
while (srcs_dup.length < numberOfFiles) {
var fileIndex = randomIndex(dirs.length-1);
//Check if the file was already added to the array
if (hashCheck[fileIndex] == true) {
continue; //Already have that file. Skip it
}
//Add the file to the array and object
srcs_dup.push(dirs[fileIndex]);
hashCheck[fileIndex] = true;
}
var chosen = "'" + srcs_dup[0] + "'"
var call = "ffmpeg -re -i " + chosen + " -content_type audio/mpeg -f mp3 udp://224.1.2.3:1234"
const stop = childProcess.exec(call, { shell: true });
stop.stdout.on('data', function (data) {
console.log('stdout: ' + data.toString());
});
stop.stderr.on('data', (data) => {
console.log(`stderr: ${data}`);
});
stop.on('close', (code) => {
console.log ('child exited with code ' + code)
Stream();
});
stop.on('error', function(err) {
console.log('sh error' + err)
});
}
I'm recording the users screen via webrtc, and then posting video blobs every x seconds using MediaStreamRecorder. On the server side I have an action set up in sails which saves the blob as a webm file.
The problem is that I can't get it to append the data, and create one large webm file. When it appends the file size increases like expected, so the data is appending, but when I go to play the file it'll either play the first second, not play at all, or play but not show the video.
It would be possible to merge the files with ffmpeg, but I'd rather avoid this if at all possible.
Here's the code on the client:
'use strict';
// Polyfill in Firefox.
// See https://blog.mozilla.org/webrtc/getdisplaymedia-now-available-in-adapter-js/
if (typeof adapter != 'undefined' && adapter.browserDetails.browser == 'firefox') {
adapter.browserShim.shimGetDisplayMedia(window, 'screen');
}
io.socket.post('/processvideo', function(resData) {
console.log("Response: " + resData);
});
function handleSuccess(stream) {
const video = document.querySelector('video');
video.srcObject = stream;
var mediaRecorder = new MediaStreamRecorder(stream);
mediaRecorder.mimeType = 'video/webm';
mediaRecorder.ondataavailable = function (blob) {
console.log("Sending Data");
//var rawIO = io.socket._raw;
//rawIO.emit('some:event', "using native socket.io");
io.socket.post('/processvideo', {"vidblob": blob}, function(resData) {
console.log("Response: " + resData);
});
};
mediaRecorder.start(3000);
}
function handleError(error) {
errorMsg(`getDisplayMedia error: ${error.name}`, error);
}
function errorMsg(msg, error) {
const errorElement = document.querySelector('#errorMsg');
errorElement.innerHTML += `<p>${msg}</p>`;
if (typeof error !== 'undefined') {
console.error(error);
}
}
if ('getDisplayMedia' in navigator) {
navigator.getDisplayMedia({video: true})
.then(handleSuccess)
.catch(handleError);
} else {
errorMsg('getDisplayMedia is not supported');
}
Code on the server:
module.exports = async function processVideo (req, res) {
var fs = require('fs'),
path = require('path'),
upload_dir = './assets/media/uploads',
output_dir = './assets/media/outputs',
temp_dir = './assets/media/temp';
var params = req.allParams();
if(req.isSocket && req.method === 'POST') {
_upload(params.vidblob, "test.webm");
return res.send("Hi There");
}
else {
return res.send("Unknown Error");
}
function _upload(file_content, file_name) {
var fileRootName = file_name.split('.').shift(),
fileExtension = file_name.split('.').pop(),
filePathBase = upload_dir + '/',
fileRootNameWithBase = filePathBase + fileRootName,
filePath = fileRootNameWithBase + '.' + fileExtension,
fileID = 2;
/* Save all of the files as different files. */
/*
while (fs.existsSync(filePath)) {
filePath = fileRootNameWithBase + fileID + '.' + fileExtension;
fileID += 1;
}
fs.writeFileSync(filePath, file_content);
*/
/* Appends the binary data like you'd expect, but it's not playable. */
fs.appendFileSync(upload_dir + '/' + 'test.file', file_content);
}
}
Any help would be greatly appreciated!
I decided this would be difficult to develop, and wouldn't really fit the projects requirements. So I decided to build an electron app. Just posting this so I can resolve the question.
I have a firebase function that will take a request from frontend with a file's name, which will be a video that stored in firebase storage, and then I will apply ffmpeg and extract the video to many frames. In the end, I will upload all frames into firebase storage.
Everything works good, I am able to get all frames. However, there is a problem with uploading frames. Sometimes I can upload all frames successfully, but the function will keep running until timeout, and sometimes I can only upload the first frame. I am new to node.js. I guess there is a problem with return or promise (I don't quit understand what to return and how to handle promise).
Also, I would like to write the data of each frame to database. Where should I put this part of code?
exports.extractFrame = functions.https.onRequest(function (req, res) {
const name = req.query.fileName;
const username = name.substr(0, name.length - 4);
const sessionId = 'video-org';
const framePath = 'frame-org';
const sourceBucketName = 'this is my bucket name';
const sourceBucket = gcs.bucket(sourceBucketName);
const temDir = os.tmpdir();
return sourceBucket.file(sessionId + '/' + name).download({
destination: temDir + '/' + name
}
).then(() => {
console.log('extract frames');
return spawn(ffmpegPath, ['-i', temDir + '/' + name, temDir + '/' +
username + '%d.png']);
}).then(() => {
const frames = fs.readdirSync(temDir);
console.log(frames);
for (let index in frames) {
if (index != 0) {
console.log('uploading');
sourceBucket.upload(temDir + '/' + frames[index], {destination:
framePath + '/' + frames[index]});
}
}
}).then(() => {
res.send('I am done');
});
});
Thanks so much for the help!!
Collect all the promises from all of the calls to sourceBucket.upload() into an array, then use Promise.all() to wait for the entire set to resolve before sending the response:
const promises = [];
for (let index in frames) {
if (index != 0) {
console.log('uploading');
const p = sourceBucket.upload(temDir + '/' + frames[index], {destination:
framePath + '/' + frames[index]});
promises.push(p);
}
}
return Promise.all(promises);
Also, you don't return a promise from an HTTP type function. Just sending the response with res.send() will end the function. This is mentioned in the documentation.
I wrote a gist on this a while back:
// set it up
firebase.storage().ref().constructor.prototype.putFiles = function(files) {
var ref = this;
return Promise.all(files.map(function(file) {
return ref.child(file.name).put(file);
}));
}
// use it!
firebase.storage().ref().putFiles(files).then(function(metadatas) {
// Get an array of file metadata
}).catch(function(error) {
// If any task fails, handle this
});
I'm trying to populate a database with pictures paths & names using Node.js.
What I am trying to do is the following :
- A function send a list of pictures as Base64 string.
- Another function receive this list, loop through it, convert it into picture and get the path back.
I'm pretty new to node.js so I might be doing something really stupid.
Here is the reception code :
app.post('/chatBot/moreinfo/create', function (req, res) {
returnList = '';
res.set('Content-Type', 'application/json');
//IF LIST IS NOT EMPTY
if (req.body.imgList.length !== 0) {
const imagePath = '/var/lib/SMImageBank/';
const regex = /^data:image\/(.*);.*$/i;
const listePicture = req.body.imgList;
// LOOPING INTO THE LIST
req.body.imgList.map ( function (element) {
const file = element;
const filetype = file.match(regex)[1];
var picLink2 = '';
const base64data = file.replace(/^data:image\/.*;base64,/, "");
const latin1data = new Buffer(base64data, 'base64').toString('latin1');
const filename = new Date().getTime() + '' + new Date().getMilliseconds() + "." + filetype;
fs.mkdir(imagePath, () => {
fs.writeFile(imagePath + filename, latin1data, "latin1", function (err, content) {
if (err) {
routerLog(req, {'type': 'error', 'content': err} );
res.sendStatus(500);
}
else {
if (process.env.NODE_ENV === "production")
picLink2 = 'http://****.fr/image/' + filename;
else if(process.env.NODE_ENV === "test")
picLink2 = 'http://dev.****.fr:8010/image/' + filename;
else if(process.env.NODE_ENV === "master")
picLink2 = 'http://dev.****.fr:8008/image/' + filename;
else{
picLink2 = 'http://*****.com:8008/image/' + filename;
}
}
});
})
console.log(picLink2);
returnList = returnList + ";" + picLink2;
});
}
MoreInfo.create(req.body, function (ret) {
res.send(ret);
routerLog(req);
})
});
What I want to do is to be able to access the variable "picLink2" from outside the writeFile & mkdir function so I can populate my returnList at each iteration. Obviously as node.js is asynchronous I can't access to picLink2 content from outside fs.writeFile() function. I know there has been a ton of question about this and lot of the answers are to put the code inside the writeFile()/readFile() function but I don't see how I can do it here since the writeFile() function is inside a map that is iterating into a list.
I'm new to the asynchronous world and I don't see how I can solve this problem.
Use writeFileSync for a synchronous operation if that doesn't hurt performance.