I'm trying to populate a database with pictures paths & names using Node.js.
What I am trying to do is the following :
- A function send a list of pictures as Base64 string.
- Another function receive this list, loop through it, convert it into picture and get the path back.
I'm pretty new to node.js so I might be doing something really stupid.
Here is the reception code :
app.post('/chatBot/moreinfo/create', function (req, res) {
returnList = '';
res.set('Content-Type', 'application/json');
//IF LIST IS NOT EMPTY
if (req.body.imgList.length !== 0) {
const imagePath = '/var/lib/SMImageBank/';
const regex = /^data:image\/(.*);.*$/i;
const listePicture = req.body.imgList;
// LOOPING INTO THE LIST
req.body.imgList.map ( function (element) {
const file = element;
const filetype = file.match(regex)[1];
var picLink2 = '';
const base64data = file.replace(/^data:image\/.*;base64,/, "");
const latin1data = new Buffer(base64data, 'base64').toString('latin1');
const filename = new Date().getTime() + '' + new Date().getMilliseconds() + "." + filetype;
fs.mkdir(imagePath, () => {
fs.writeFile(imagePath + filename, latin1data, "latin1", function (err, content) {
if (err) {
routerLog(req, {'type': 'error', 'content': err} );
res.sendStatus(500);
}
else {
if (process.env.NODE_ENV === "production")
picLink2 = 'http://****.fr/image/' + filename;
else if(process.env.NODE_ENV === "test")
picLink2 = 'http://dev.****.fr:8010/image/' + filename;
else if(process.env.NODE_ENV === "master")
picLink2 = 'http://dev.****.fr:8008/image/' + filename;
else{
picLink2 = 'http://*****.com:8008/image/' + filename;
}
}
});
})
console.log(picLink2);
returnList = returnList + ";" + picLink2;
});
}
MoreInfo.create(req.body, function (ret) {
res.send(ret);
routerLog(req);
})
});
What I want to do is to be able to access the variable "picLink2" from outside the writeFile & mkdir function so I can populate my returnList at each iteration. Obviously as node.js is asynchronous I can't access to picLink2 content from outside fs.writeFile() function. I know there has been a ton of question about this and lot of the answers are to put the code inside the writeFile()/readFile() function but I don't see how I can do it here since the writeFile() function is inside a map that is iterating into a list.
I'm new to the asynchronous world and I don't see how I can solve this problem.
Use writeFileSync for a synchronous operation if that doesn't hurt performance.
Related
I am trying to zip a single file using the Archiver npm package located: https://www.npmjs.com/package/archiver
I have been able to use the following to zip a directory:
archive.directory(folderName, false);
But when I try to use either of these nothing seems to happen (ie: no zip is generated, file never finishes zipping):
archive.file(folderName, { name: 'file4.txt' });
archive.file(fs.createReadStream(path.resolve(file)), {name: 'File' + singleFileCheck});
Has anyone run into this issue before? Please let me know what I am doing wrong. Thank you in advance!
edit:
module.exports = async function zipper(user, pass, orgid, s4url, apiToken, newOrgName, file) {
const s4 = require('../testcli/s4');
const fs = require('fs');
const archiver = require('archiver');
const path = require('path');
var parentDirect;
if(file == "./"){
parentDirect = "..";
}else{
parentDirect = path.basename(path.dirname(file));
}
const newZipFile = parentDirect + '/s4.zip';
var folderName = file;
//Checks for existence of infinite loop
if(path.resolve(parentDirect).length > path.resolve(folderName).length){
console.log(folderName.search(parentDirect));
console.error('\x1b[36m%s\x1b[0m', 'ERROR!!!! : Please adjust where your console is pointed, this will result in an infinite loop. Exiting.');
return;
}
var P = ['\\', '|', '/', '-'];
var x = 0;
var output = fs.createWriteStream(newZipFile);
var archive = archiver('zip');
scansdisplayinterval = setInterval(function () {
twrl();
}, 250);
// listen for all archive data to be written
output.on('close', function () {
console.log('\x1b[36m%s\x1b[0m', archive.pointer() + ' total bytes');
console.log('\x1b[36m%s\x1b[0m', 'archiver has been finalized and the output file descriptor has closed.');
try {
process.stdout.write(newZipFile);
clearInterval(scansdisplayinterval);
s4(user, pass, newZipFile, orgid, s4url, apiToken, newOrgName);
} catch (e) {
console.log(e);
}
});
// good practice to catch this error explicitly
archive.on('error', function (err) {
throw err;
});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on('warning', function(err) {
throw err;
});
// This event is fired when the data source is drained no matter what was the data source.
output.on('end', function() {
console.log('\x1b[36m%s\x1b[0m', 'Data has been drained');
});
// pipe archive data to the file
archive.pipe(output);
//Checks -f for file extension
let singleFileCheck = path.extname(file);
//If file has extension
if(singleFileCheck.length <= 4 && singleFileCheck != ''){
//Append single file
console.log('singleFile', path.resolve(file));
archive.file(path.resolve(file), { name: 'file4.txt' });
// archive.append(fs.createReadStream(path.resolve(file)), {name: 'File' + singleFileCheck});
//Else = folder
}else{
// append files from a sub-directory, putting its contents at the root of archive
archive.directory(folderName, false);
}
// archive.directory(folderName, false);
console.log('\x1b[36m%s\x1b[0m', "Zipping: " + folderName + " To: " + newZipFile);
console.log('\x1b[36m%s\x1b[0m', "Zipping To: " + path.resolve(newZipFile));
archive.finalize();
function twrl() {
process.stdout.write('\rZipping Folder ... ' + P[x++]);
x &= 3;
}
return(newZipFile);
};
The issue came from how I was defining the parentDirect var.
Solution:
let singleFileCheck = path.extname(file);
if(file == "./" || singleFileCheck.length <= 4 && singleFileCheck != ''){
parentDirect = "..";
}else{
parentDirect = path.basename(path.dirname(file));
}
I have an implementation of a POST router in node.js with express. The body of the post is a list of files, and the idea is that these files should be zipped and a link to the resulting zip file should be returned. I'm trying to use 'express-session' to store the progress in terms of number of files zipped as well as the file name of the zip upon completion. Updating the number of files zipped (req.session.current) works fine, but for some reason I can't set the file name (req.session.zipFile) in the session when the fileStream closes. I'm suspecting that the req object is no longer valid by the time fileStream.on('close') is reached, but I'm not sure how to handle that. I can't access the session data without a request.
This is the complete router implementation:
const express = require('express');
const router = express.Router();
const fs = require('fs');
const ZIP_FILES_PATH = require('../constants');
router.post('/', function(req, res, next){
const json = JSON.parse(req.body.data);
const path = json["base"];
if (!path || path.length === 0){
res.render("error", {msg: "JSON missing 'base'"})
return;
}
const files = json["files"];
if (!files || files.length === 0){
res.statusCode = 400;
res.send("JSON missing 'files'");
return;
}
if (!fs.existsSync(path)) {
res.statusCode = 400;
res.send("Directory '" + path + "' does not exist");
return;
}
try{
var sessData = req.session;
sessData.total = files.length ;
sessData.current = 0;
sessData.zipFile = '';
zipFiles(path, files, req, res);
}catch(error){
res.statusCode = 500;
res.send("The files could not be zipped");
return;
}
res.render("zipping")
});
module.exports = router;
const zipFiles = (path, files, req, res) => {
const zipFile = require('crypto').createHash('md5').update(path).digest("hex") + "_" + new Date().getTime() + ".zip";
const archiver = require('archiver');
const fileStream = fs.createWriteStream(ZIP_FILES_PATH + zipFile);
const archive = archiver('zip', {
gzip: true,
zlib: { level: 9 }
});
archive.on('error', function(err) {
throw err;
});
fileStream.on('close', function() {
req.session.zipFile = "/download?file=" + zipFile; //Doesn't stick!
});
archive.pipe(fileStream);
files.map(file => zipSingleFile(path, file, req, archive));
archive.finalize();
}
const zipSingleFile = (path, file, req, archive) => {
fs.existsSync(path + "/" + file) ? archive.file(path + "/" + file, { name: file }) : null;
req.session.current = req.session.current + 1;
}
zipFiles() is asynchronous. That means it doesn't block and finishes some time later. So, in the flow of your request handler, you call
res.render("zipping")
before zipFiles() finishes. And, res.render() will send the response and close the output stream.
If you want to wait to call res.render() until you're done with the zipFiles() call, then you need to either call it from within zipFiles() when it's all done or you need to add a callback to zipFiles() so it can communicate back when it's done.
Also, is there any reason you can't set req.session.zipFile = "/download?file=" + zipFile; before you call res.render()?
Also, depending upon how you have your session configured (which you don't show us any code for), just doing this:
req.session.zipFile = "/download?file=" + zipFile;
may not be enough to actually save that value into the session. You may also have to call .save() on the session.
And, you are setting:
req.session.zipFile
at the end of your zipping process so any requests that arrive from that session before your zipping finishes won't see that session variable set either (this seems like a concurrency problem to me).
I'm recording the users screen via webrtc, and then posting video blobs every x seconds using MediaStreamRecorder. On the server side I have an action set up in sails which saves the blob as a webm file.
The problem is that I can't get it to append the data, and create one large webm file. When it appends the file size increases like expected, so the data is appending, but when I go to play the file it'll either play the first second, not play at all, or play but not show the video.
It would be possible to merge the files with ffmpeg, but I'd rather avoid this if at all possible.
Here's the code on the client:
'use strict';
// Polyfill in Firefox.
// See https://blog.mozilla.org/webrtc/getdisplaymedia-now-available-in-adapter-js/
if (typeof adapter != 'undefined' && adapter.browserDetails.browser == 'firefox') {
adapter.browserShim.shimGetDisplayMedia(window, 'screen');
}
io.socket.post('/processvideo', function(resData) {
console.log("Response: " + resData);
});
function handleSuccess(stream) {
const video = document.querySelector('video');
video.srcObject = stream;
var mediaRecorder = new MediaStreamRecorder(stream);
mediaRecorder.mimeType = 'video/webm';
mediaRecorder.ondataavailable = function (blob) {
console.log("Sending Data");
//var rawIO = io.socket._raw;
//rawIO.emit('some:event', "using native socket.io");
io.socket.post('/processvideo', {"vidblob": blob}, function(resData) {
console.log("Response: " + resData);
});
};
mediaRecorder.start(3000);
}
function handleError(error) {
errorMsg(`getDisplayMedia error: ${error.name}`, error);
}
function errorMsg(msg, error) {
const errorElement = document.querySelector('#errorMsg');
errorElement.innerHTML += `<p>${msg}</p>`;
if (typeof error !== 'undefined') {
console.error(error);
}
}
if ('getDisplayMedia' in navigator) {
navigator.getDisplayMedia({video: true})
.then(handleSuccess)
.catch(handleError);
} else {
errorMsg('getDisplayMedia is not supported');
}
Code on the server:
module.exports = async function processVideo (req, res) {
var fs = require('fs'),
path = require('path'),
upload_dir = './assets/media/uploads',
output_dir = './assets/media/outputs',
temp_dir = './assets/media/temp';
var params = req.allParams();
if(req.isSocket && req.method === 'POST') {
_upload(params.vidblob, "test.webm");
return res.send("Hi There");
}
else {
return res.send("Unknown Error");
}
function _upload(file_content, file_name) {
var fileRootName = file_name.split('.').shift(),
fileExtension = file_name.split('.').pop(),
filePathBase = upload_dir + '/',
fileRootNameWithBase = filePathBase + fileRootName,
filePath = fileRootNameWithBase + '.' + fileExtension,
fileID = 2;
/* Save all of the files as different files. */
/*
while (fs.existsSync(filePath)) {
filePath = fileRootNameWithBase + fileID + '.' + fileExtension;
fileID += 1;
}
fs.writeFileSync(filePath, file_content);
*/
/* Appends the binary data like you'd expect, but it's not playable. */
fs.appendFileSync(upload_dir + '/' + 'test.file', file_content);
}
}
Any help would be greatly appreciated!
I decided this would be difficult to develop, and wouldn't really fit the projects requirements. So I decided to build an electron app. Just posting this so I can resolve the question.
I have a firebase function that will take a request from frontend with a file's name, which will be a video that stored in firebase storage, and then I will apply ffmpeg and extract the video to many frames. In the end, I will upload all frames into firebase storage.
Everything works good, I am able to get all frames. However, there is a problem with uploading frames. Sometimes I can upload all frames successfully, but the function will keep running until timeout, and sometimes I can only upload the first frame. I am new to node.js. I guess there is a problem with return or promise (I don't quit understand what to return and how to handle promise).
Also, I would like to write the data of each frame to database. Where should I put this part of code?
exports.extractFrame = functions.https.onRequest(function (req, res) {
const name = req.query.fileName;
const username = name.substr(0, name.length - 4);
const sessionId = 'video-org';
const framePath = 'frame-org';
const sourceBucketName = 'this is my bucket name';
const sourceBucket = gcs.bucket(sourceBucketName);
const temDir = os.tmpdir();
return sourceBucket.file(sessionId + '/' + name).download({
destination: temDir + '/' + name
}
).then(() => {
console.log('extract frames');
return spawn(ffmpegPath, ['-i', temDir + '/' + name, temDir + '/' +
username + '%d.png']);
}).then(() => {
const frames = fs.readdirSync(temDir);
console.log(frames);
for (let index in frames) {
if (index != 0) {
console.log('uploading');
sourceBucket.upload(temDir + '/' + frames[index], {destination:
framePath + '/' + frames[index]});
}
}
}).then(() => {
res.send('I am done');
});
});
Thanks so much for the help!!
Collect all the promises from all of the calls to sourceBucket.upload() into an array, then use Promise.all() to wait for the entire set to resolve before sending the response:
const promises = [];
for (let index in frames) {
if (index != 0) {
console.log('uploading');
const p = sourceBucket.upload(temDir + '/' + frames[index], {destination:
framePath + '/' + frames[index]});
promises.push(p);
}
}
return Promise.all(promises);
Also, you don't return a promise from an HTTP type function. Just sending the response with res.send() will end the function. This is mentioned in the documentation.
I wrote a gist on this a while back:
// set it up
firebase.storage().ref().constructor.prototype.putFiles = function(files) {
var ref = this;
return Promise.all(files.map(function(file) {
return ref.child(file.name).put(file);
}));
}
// use it!
firebase.storage().ref().putFiles(files).then(function(metadatas) {
// Get an array of file metadata
}).catch(function(error) {
// If any task fails, handle this
});
I'm currently having a problem when trying to download many images through a external request.get, bellow are the code
var future = new Future();
var images, nome, blob;
.each(imoveis, function(dadosImovel, numeroImovel){
images = dadosImovel.images;
_.each(imagens, function(value, key){
// the name of the image to a permalink format, this function is working
nome = Meteor.createPermalinkFromString(value[3]);
// the link pointing to the image
blob = Meteor.getImage(value[0]);
Meteor.saveImage(blob, nome, '.jpeg');
});
});
// Get a image through a url
Meteor.getImage = function(url){
var options = {
url : url,
encoding : null
};
// get raw image
request.get(options, function(error, result, body) {
if (error) {
return console.error(error);
}
// pause until binaries are fully loaded
future['return'](body);
});
return future.wait()
};
// save a image in a server folder
Meteor.saveImage = function(name, blob, ext, encoding) {
var projectPath = basepath.resolve('.').split('.meteor')[0],
chroot = Meteor.chroot || projectPath + 'public', // (process.env['PWD'] +'/public') ;
path = chroot + (path ? '/' + path + '/' : '/'),
name = Meteor.cleanName(name || 'file'),
encoding= encoding || 'binary';
// TODO Add file existance checks, etc...
fs.writeFile(path + name + ext, blob, encoding, function(err) {
if (err) {
console.log(err);
throw (new Meteor.Error(500, 'Failed to save file.', err));
} else {
console.log('The file ' + name + ' (' + encoding + ') was saved to ' + path);
}
});
return true;
}
Now here what happens: in the first iteration of the loop I receive the image just fine, the problem appears for the next ones iterations.
If I have, 10 images all of then are saved with 10 different names (and they also have 10 links pointing to the correct images link) but when you visualize the image it's the first image on the list save for all the other 9, it's like once the first image is loaded meteor don't wait for the next binary/image codes to arrive before saving then.
Any thoughts on how to fix this?
Changing the
var Future = new Future();
from the outter scope to inside the function getImage fixed the issue with the images, but created another one.
My current code is
_.each(imoveis, function(dadosImovel, numeroImovel){
var imagens = dadosImovel.imagens;
_.each(imagens, function(value, key){
var nome = Meteor.createPermalinkFromString(value[3]);
var blob = Meteor.getImage(value[0]);
Meteor.saveImage(blob, nome, '.jpeg');
});
});
And now I have a infinite outer loop, saving again and again only the set of images from the first item of the loop.
For referente (in case this might help someone in the future) this is the code I used to make this works:
// imovel[0] = link, imovel[1] = width, imovel[2] = height, imovel[3] = title, imovel[4] = codigo
var futures = _.map(links, function(imovel) {
var future = new Future();
var onComplete = future.resolver();
var options = {
url : imovel[0],
encoding : null
};
// get raw image
request.get(options, function(error, result, body) {
if (error) {
return console.error(error);
}
var nome = Meteor.createPermalinkFromString(imovel[3]);
var data = {'codigo' : imovel[4], 'blob' : body, 'nome' : nome};
onComplete(error, data);
});
return future;
});
// wait for all futures to finish
Future.wait(futures);
// and grab the results out.
links = _.invoke(futures, 'get');