node.js move bunch of files - node.js

i have made a form for file upload and i have set the multiple option, so im trying to upload a bunch of files and then move them according to the album name that the client have set,
here is what iv done:
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
var file_name = null;
switch(req.files.img_file.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
default:
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ לא תקין'
});
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
else
{
_.each(req.files.img_file,function(val,index){
console.log(val.path + " " + index);
//gives the file path so i can read it
fs.readFile(val.path, function (err, data) {
if (err){
console.log("fs " + err);
}
//so until here everything works fine, the files are uploaded to the "/uploads" directory, now im trying to move them to the correct album, the destiation is : public/img/albums/:album_name/:all_images here
mv(val.path, './public/img/albums/' + album_name + '/' + val.path, function(err) {
if (err){
console.log("mv " + err);
}
else{
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה'
});
res.redirect('/admin');
}
});
});
});
}
});
the mv module throws an error rename "c:/work/xxx/xx/uploads/val.path.png

Its a file-access error. You X out some of the file name, but look at how that NPM module handles file, and ensure you are naming the files and paths properly. Then it should work out fine.

i have used read and write stream and deleted the "mv" module
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
if (req.files.img_file.length > 1)
{
var Counter = 0;
_.each(req.files.img_file,function(val,index){
var file_name = null;
switch(val.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
var source = fs.createReadStream(val.path);
var dest = fs.createWriteStream('./public/img/albums/' + album_name + '/' + val.name);
source.pipe(dest);
source.on('end', function() {
console.log('end...');
Counter++;
console.log(Counter);
console.log(req.files.img_file.length);
if (Counter == req.files.img_file.length){
res.redirect('/admin');
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה',
albums: albums
}); //eo res render
}
});
source.on('error', function(err) { console.log('error'); });
});// eo mkdir
}); // eo _each
}

Related

NodeJS itself keeps file EBUSY on Windows?

I created a simple function to process uploaded files. I'm using multer to process the multipart data into files. Then I use the code below to move the files around, and return data so my webpage knows how to display the images.
It seems that somehow NodeJS keeps the files open itself. I also created a function to remove the files, but this will give me an EBUSY error. If I try to remove through Windows, it says that NodeJS has te file locked. When I restart the NodeJS process and then re-request the delete URL, the file is removed correctly.
Is there some way I can force NodeJS to close the file resources? Or is there some other error in my script that I am missing?
I updated node to version 12.4.0 but this didn't help either.
Processing the uploads:
exports.handleFormNotes = async(req, res, next) => {
try {
const configVariables = req.app.get('configVariables');
const uploadSuffix = req.body.uploadFolderSuffix || '';
console.log('upload suffix', uploadSuffix);
if (!req.files.length) {
return;
}
const uploadedFiles = Array();
var destPath = configVariables['FormNotesUploadDirectory'];
if (uploadSuffix !== '')
destPath = destPath + '/' + uploadSuffix;
destPath = path.resolve(destPath);
// mkdirSync returns undefined, so run that first and see if the directory exists second.
if (!fs.mkdirSync(destPath, { recursive: true }) && !fs.existsSync(destPath)) {
console.log(destPath, 'does not exist!');
req.alertHandler.addAlert('Pad om afbeelding op te slaan is niet bereikbaar: ' + destPath, 'danger');
res.render('error');
return;
}
var baseUrlPath = configVariables['FormNotesUploadDocumentRoot'];
if (uploadSuffix != null) {
baseUrlPath = baseUrlPath + '/' + uploadSuffix;
}
for(const uploadedFile of req.files) {
let now = new Date();
let destFilename = getDateTime() + "_" + uploadedFile.originalname;
let destFilenameThumb = 'thumb_' + destFilename;
var fullDestination = path.resolve(destPath + '/' + destFilename);
var fullDestinationThumb = path.resolve(destPath + '/' + destFilenameThumb);
console.log('Copy src:', uploadedFile.path, fullDestination);
fs.copyFileSync(uploadedFile.path, fullDestination);
var unlinkResult = fs.unlinkSync(uploadedFile.path);
console.log('Unlink "' + uploadedFile.path + '", result after upload:', unlinkResult);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);
console.log('new file info thumb:', newFileInfo);
uploadedFiles.push({
'fullImg': baseUrlPath + '/' + destFilename,
'thumbImg' : baseUrlPath + '/' + destFilenameThumb,
'original': uploadedFile.originalname
});
}
// Push to backend
const data = {
files: [...uploadedFiles],
uploadSuffix: uploadSuffix
};
// Normally retVal should be the return data from OI. If anything goes wrong, retVal = 'error'
this.saveAttachment(req, res, data);
return res.send(data);
}
catch (err) {
console.log('Error handling from notes:', err);
req.alertHandler.addAlert('Error handling form notes: ' + err);
return 'error';
}
}
Removing the uploads:
exports.rmFormNote = async(req, res, data) => {
let retVal;
try {
const configVariables = req.app.get('configVariables');
const httpPath = req.query.img;
console.log('http path:', httpPath);
// Strip off the document root, but check if they are the same first
const firstPart = httpPath.substring(0, configVariables['FormNotesUploadDocumentRoot'].length);
console.log('same?', firstPart, configVariables['FormNotesUploadDocumentRoot']);
var relPath = httpPath;
if (firstPart == configVariables['FormNotesUploadDocumentRoot']) {
relPath = httpPath.substring(configVariables['FormNotesUploadDocumentRoot'].length + 1);
}
var parts = relPath.split('/');
parts[parts.length-1] = 'thumb_' + parts[parts.length-1];
var thumbPath = parts.join('/');
thumbPath = path.resolve(configVariables['FormNotesUploadDirectory'] + '/' + thumbPath);
console.log('thumbpath: ', thumbPath);
var fullPath = configVariables['FormNotesUploadDirectory'] + '/' + relPath;
var dest = path.resolve(fullPath);
console.log('dest: ', dest);
if (!fs.existsSync(dest))
throw "File not found";
fs.unlink(dest, (err) => {
if (err) throw err;
console.log('File deleted');
});
retVal = { result: true };
}
catch(err) {
console.log('Ohnoo', err);
retVal = { result: false, msg: err };
}
return res.send(retVal);
}
Turns out the thumbnail creator sharp was the problem, as stated in this github issue.
I just had to disable the cache, like so:
sharp.cache(false);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);

Nodejs bluebird promise fails while processing image

//Created a promise for each image size.
var promises = sizes.map(function (size) {
return new Promise(function (resolve, reject) {
var destinationDir = fileUtil.getAbsolutePathOfImage(destinationPath);
fileUtil.createDirectoryIfNotExists(destinationDir);
destinationDir += size.src;
fileUtil.createDirectoryIfNotExists(destinationDir);
//Resize the image.
//console.log('imagefile : ' + JSON.stringify(imageFile));
//console.log('destinationDir: ' + JSON.stringify(destinationDir));
//Called an imageUtil resize method to perform resize.
imageUtil.resize(imageFile, destinationDir, size).then(data => {
var fileName = destinationPath + size.src + '/' + data;
resolve(imageUtil.createImageData(fileName, size.height, size.width));
}).catch(err => {
console.error(err);
return reject(err);
});
});
});
Promise.all(promises)
.then(savedImages => {
console.log('saved Images are: ' + JSON.stringify(savedImages));
return res.status(200).json(savedImages);
}).catch(err => {
console.log('i am here' + JSON.stringify(err.message));
return res.status(400).json(JSON.stringify(err.message));
});
---------------------Resize method of imageutil---------------
var Promise = require('bluebird'),
gm = require('gm'),
path = require('path'),
fs = require('fs');
Promise.promisifyAll(gm.prototype);
module.exports = {
resize(imageFile, destinationPath, size){
if (!imageFile || !destinationPath || !size) {
return;
}
return new Promise(function (resolve, reject) {
// If we just passed callback directly, errors would be fatal
var fileName = fileUtil.getFileName(imageFile);
//console.log('sourceFile : ' + JSON.stringify(imageFile));
//console.log('saveDirectory : ' + JSON.stringify(destinationPath));
//console.log('fileName is :' + fileName);
//Create a write stream.
var writeStream = fs.createWriteStream(destinationPath + '/' + fileName);
//console.log('Saving at location: ' + writeStream.path);
gm(imageFile)
.resize(size.width, size.height, '^')
.gravity('Center')
.crop(size.width, size.height)
.writeAsync(writeStream.path, function (err) {
if (err) {
var error = 'Error while creating image of resolution : ' + size.width + 'x' + size.height + '.';
console.error(JSON.stringify(error));
return reject(new Error(error));
}
});
resolve(fileName);
});
}
};
*It seems like everything went correct and it creates four image file which is corrupted and gave me error later on but request processed succesfully. Console output of my image processing are as follows:
saved Images are: [{"src":"/uploads/300/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"200","width":"300"},{"src":"/uploads/120/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"120","width":"120"},{"src":"/uploads/48/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"48","width":"48"}]
POST /api/upload/image/ 200 51.790 ms - 241
"Error while creating image of resolution : 120x120."
"Error while creating image of resolution : 48x48."
"Error while creating image of resolution : 300x200."*
Since you are already using promisifyAll, you don't need to (and should not) use the Promise constructor. writeAsync already returns a promise - if you don't pass a callback, as that's the requirement for Bluebird being able to pass the callback itself in the right position.
You should be using
module.exports = {
resize(imageFile, destinationPath, size){
if (!imageFile || !destinationPath || !size) {
return Promise.reject(new Error("missing arguments"));
}
var fileName = fileUtil.getFileName(imageFile);
//console.log('sourceFile : ' + JSON.stringify(imageFile));
//console.log('saveDirectory : ' + JSON.stringify(destinationPath));
//console.log('fileName is :' + fileName);
//Create a write stream.
var writeStream = fs.createWriteStream(destinationPath + '/' + fileName);
//console.log('Saving at location: ' + writeStream.path);
var promise = gm(imageFile)
.resize(size.width, size.height, '^')
.gravity('Center')
.crop(size.width, size.height)
.writeAsync(writeStream.path);
return promise.then(function() {
return filename;
}, function (err) {
var error = 'Error while creating image of resolution : ' + size.width + 'x' + size.height + '.';
console.error(error, err);
throw new Error(error);
});
}
};
Similarly, you shouldn't use the Promise constructor antipattern in the call to resize - it already returns a promise:
var promises = sizes.map(function (size) {
var destinationDir = fileUtil.getAbsolutePathOfImage(destinationPath);
fileUtil.createDirectoryIfNotExists(destinationDir);
destinationDir += size.src;
fileUtil.createDirectoryIfNotExists(destinationDir);
//Resize the image.
//console.log('imagefile : ' + JSON.stringify(imageFile));
//console.log('destinationDir: ' + JSON.stringify(destinationDir));
return imageUtil.resize(imageFile, destinationDir, size)
//^^^^^^
.then(data => {
var fileName = destinationPath + size.src + '/' + data;
return imageUtil.createImageData(fileName, size.height, size.width));
}, err => {
console.error(err);
throw err;
});
});

local variable is not changing

I'm using node.js for converting text files to CSV. It works for one file, but when i try process more files fileDestination variable doesn't change. Why? Input files like this: r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
I got following console output:
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_1_5X1x9_05_train2.res
/*
* lee archivos *.dat y los convierte *.csv
*/
const fs = require('fs');
const inputDir = './';
const outputDir = './';
function readFiles(inputDir, onError) {
fs.readdir(inputDir, function(err, filenames) {
if (err) {
onError(err);
return;
}
filenames.forEach(function(inputFile) {
// first we arre looking for "right" file name
if (inputFile.search(/res/) != -1) {
console.log('Starting processing ' + inputFile);
convert2csv(inputFile, function(error) {
throw err;
});
}
});
});
}
function convert2csv(filename, onError) {
arrayFromFilename = filename.split('_');
epoca = arrayFromFilename[4];
trainORval = arrayFromFilename[7].replace('2.res', '');
console.log("from convert " + filename + " " + epoca);
fs.readFile(inputDir + filename, 'utf-8', function(err, content) {
if (err) {
onError(err);
return;
}
content = content.replace(/^[^0].*\n/mg, '');
arr = content.split('\n');
pares = arr.filter(function(d, i) {
return i % 2 == 1;
});
content = pares.join('\n');
content = content.replace(/(^[\d.]*) ([\d.]*)/gm, '$1,$2');
fileDestination = outputDir + epoca + '_' + trainORval + '.csv';
console.log("filedestination :" + fileDestination);
fs.writeFile(fileDestination, 'y,x\n', function(err) {
if (err) {
return console.error(err);
}
fs.appendFile(fileDestination, content, function(err) {
if (err) {
return console.error(err);
}
console.log(fileDestination + " has been written successufully!", filename);
});
});
});
}

NodeJS download remote to local FTP

I'm trying to download a remote location on a FTP server to my local directory. I've been using the jsftp module (https://github.com/sergi/jsftp) for this, but I'm running into a problem.
Basically I'm trying to recursively download the whole remote location, including all the sub-folders etc. I've been trying to figure it out myself, but so far no luck.
so far I've been trying this:
var worker = {
ftp: null,
init: function() {
this.ftp = new JSFtp({
host: "localhost",
port: 21,
user: "username",
pass: "password"
});
this.ftp.auth("username", "password", function (result) {
if (typeof result !== "undefined") {
console.log("Something went wrong");
}
this.handleData();
}.bind(this));
},
handleData: function () {
recursive_get_files(this, "/");
}
};
function recursive_get_files(worker, dir) {
console.log("Getting directory: " + dir);
worker.ftp.ls(dir, function (err, res) {
res.forEach(function (file) {
if (file.type === 1) {
recursive_get_files(worker, dir + "/" + file.name);
} else {
worker.ftp.get(dir + "/" + file.name, "/downloads" + dir + "/" + file.name, function(err) {
if (err) {
console.log("Couldn't download file: " + file.name);
console.log(err);
}
}.bind(file.name));
}
});
});
}
The biggest problem, I think, is that all these get functions are getting called almost instantly after each other, and since the client is probably not allowed to call so many things, it will break.
I've seen a module named ftpsync (https://www.npmjs.com/package/ftpsync) do some remote syncing, from local to remote, but I need this the otherway around.
is anyone able to help me here? I've been stuck on this all day =/.
I was able to fix this with a friend of mine by doing the following:
function recursive_get_files(workerf, dir) {
total_pending++;
var worker = workerf;
var done = function() {
//probably callback here if needed
console.log("finished downloading ftp");
wrench.chmodSyncRecursive('downloads', 0777);
worker.respond(JSON.stringify({type: "success", message: "Finished"}));
};
var func_download = function() {
//Is the download queue empty?
if(download_queue.length === 0)
{
total_running--;
if(total_running === 0 && total_pending === 0)
{
done();
}
return;
}
//Get the next download in the queue
var download = download_queue[0];
download_queue.splice(0, 1);
//Get a free FTP connection
var curftp;
for(curftp = 0; curftp < total; curftp++) {
if (worker.ftp[curftp].used === false) {
worker.ftp[curftp].used = true;
break;
}
}
//Get the file
worker.ftp[curftp].ftp.get(download.dir + "/" + download.file.name, "downloads/" + worker.project + download.dir + "/" + download.file.name, function(file, err) {
worker.ftp[curftp].used = false;
if (err)
{
console.log("Couldn't download file with FTP(" + curftp + "): " + file);
console.log(err);
setTimeout(func_download, 0);
}
else
{
console.log("Downloaded file with FTP(" + curftp + "): " + file);
setTimeout(func_download, 0);
}
}.bind(null, download.file.name));
};
//Get a list of the current directory (Using the main connection)
worker.mainftp.ls(dir, function (err, res) {
res.forEach(function (file) {
if (file.type === 1)
{
mkdirp("downloads/" + worker.project + dir + "/" + file.name, function(err) {
if (err) console.log(err);
recursive_get_files(worker, dir + "/" + file.name);
});
}
else
{
download_queue.push({file: file, dir: dir});
if(total_running < total)
{
total_running++;
setTimeout(func_download, 0);
}
}
});
});
total_pending--;
if(total_running === 0 && total_pending === 0 && download_queue.length === 0)
{
done();
}
}
it uses wrench.js (for recursive chmod) and jsftp. Thanks for reading ^^

trouble using fs module to locate file

Here is my code:
connection.query("SELECT * FROM images", function(err, rows, fields) {
if (err) {
console.log("Error: ");
console.log(err);
}
else {
console.log("rows: ");
console.log(rows);
for (var i = 0; i < rows.length; i++) {
var thisRow = rows[i];
var thisImageName = thisRow["imagename"];
var newDir = __dirname.split(path.sep).join("/");
var thisImagePath = newDir + "/public/uploads/" + thisImageName + ".jpg";
console.log(thisImagePath);
fs.exists(thisImagePath, function(exists) {
if (exists) {
console.log("true");
res.end();
}
else {
console.log(thisImageName + " does not exist."); res.end();
//Always returns false
}
});
}
}
});
HOWEVER,
In the console:
fs.exists(/* imagepath */, function(exists){console.log(exists)}) //returns true
As you can see I'm using the node-mysql library to return the file names of my images. I concatenate them using the __dirname global and then I reverse the slashes in the directory string to use in the .exists() call. When I use the fs module to check if the image file exists, it returns false.
HOWEVER, if I use fs.exists() with the same path in the console, it returns true. Does anyone know what's going on here?

Resources