NodeJS itself keeps file EBUSY on Windows? - node.js

I created a simple function to process uploaded files. I'm using multer to process the multipart data into files. Then I use the code below to move the files around, and return data so my webpage knows how to display the images.
It seems that somehow NodeJS keeps the files open itself. I also created a function to remove the files, but this will give me an EBUSY error. If I try to remove through Windows, it says that NodeJS has te file locked. When I restart the NodeJS process and then re-request the delete URL, the file is removed correctly.
Is there some way I can force NodeJS to close the file resources? Or is there some other error in my script that I am missing?
I updated node to version 12.4.0 but this didn't help either.
Processing the uploads:
exports.handleFormNotes = async(req, res, next) => {
try {
const configVariables = req.app.get('configVariables');
const uploadSuffix = req.body.uploadFolderSuffix || '';
console.log('upload suffix', uploadSuffix);
if (!req.files.length) {
return;
}
const uploadedFiles = Array();
var destPath = configVariables['FormNotesUploadDirectory'];
if (uploadSuffix !== '')
destPath = destPath + '/' + uploadSuffix;
destPath = path.resolve(destPath);
// mkdirSync returns undefined, so run that first and see if the directory exists second.
if (!fs.mkdirSync(destPath, { recursive: true }) && !fs.existsSync(destPath)) {
console.log(destPath, 'does not exist!');
req.alertHandler.addAlert('Pad om afbeelding op te slaan is niet bereikbaar: ' + destPath, 'danger');
res.render('error');
return;
}
var baseUrlPath = configVariables['FormNotesUploadDocumentRoot'];
if (uploadSuffix != null) {
baseUrlPath = baseUrlPath + '/' + uploadSuffix;
}
for(const uploadedFile of req.files) {
let now = new Date();
let destFilename = getDateTime() + "_" + uploadedFile.originalname;
let destFilenameThumb = 'thumb_' + destFilename;
var fullDestination = path.resolve(destPath + '/' + destFilename);
var fullDestinationThumb = path.resolve(destPath + '/' + destFilenameThumb);
console.log('Copy src:', uploadedFile.path, fullDestination);
fs.copyFileSync(uploadedFile.path, fullDestination);
var unlinkResult = fs.unlinkSync(uploadedFile.path);
console.log('Unlink "' + uploadedFile.path + '", result after upload:', unlinkResult);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);
console.log('new file info thumb:', newFileInfo);
uploadedFiles.push({
'fullImg': baseUrlPath + '/' + destFilename,
'thumbImg' : baseUrlPath + '/' + destFilenameThumb,
'original': uploadedFile.originalname
});
}
// Push to backend
const data = {
files: [...uploadedFiles],
uploadSuffix: uploadSuffix
};
// Normally retVal should be the return data from OI. If anything goes wrong, retVal = 'error'
this.saveAttachment(req, res, data);
return res.send(data);
}
catch (err) {
console.log('Error handling from notes:', err);
req.alertHandler.addAlert('Error handling form notes: ' + err);
return 'error';
}
}
Removing the uploads:
exports.rmFormNote = async(req, res, data) => {
let retVal;
try {
const configVariables = req.app.get('configVariables');
const httpPath = req.query.img;
console.log('http path:', httpPath);
// Strip off the document root, but check if they are the same first
const firstPart = httpPath.substring(0, configVariables['FormNotesUploadDocumentRoot'].length);
console.log('same?', firstPart, configVariables['FormNotesUploadDocumentRoot']);
var relPath = httpPath;
if (firstPart == configVariables['FormNotesUploadDocumentRoot']) {
relPath = httpPath.substring(configVariables['FormNotesUploadDocumentRoot'].length + 1);
}
var parts = relPath.split('/');
parts[parts.length-1] = 'thumb_' + parts[parts.length-1];
var thumbPath = parts.join('/');
thumbPath = path.resolve(configVariables['FormNotesUploadDirectory'] + '/' + thumbPath);
console.log('thumbpath: ', thumbPath);
var fullPath = configVariables['FormNotesUploadDirectory'] + '/' + relPath;
var dest = path.resolve(fullPath);
console.log('dest: ', dest);
if (!fs.existsSync(dest))
throw "File not found";
fs.unlink(dest, (err) => {
if (err) throw err;
console.log('File deleted');
});
retVal = { result: true };
}
catch(err) {
console.log('Ohnoo', err);
retVal = { result: false, msg: err };
}
return res.send(retVal);
}

Turns out the thumbnail creator sharp was the problem, as stated in this github issue.
I just had to disable the cache, like so:
sharp.cache(false);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);

Related

Formidable couldn't parse large file

Below code works with the small files, but could upload files that is more than 50kb. I think there is something I should setting with the maxFileSize. I'm using Uppy at the client. After I console.log in the server, actually it does do the request. I don't get any clue in the Stackoverflow, really need help.
const upload = (req, res) => {
// formidable : to parse html form data
const form = new formidable.IncomingForm({ multiples: true, maxFileSize: 10000 * 1024 * 1024 })
const d = new Date();
// I have console.log here and everything seems fine
form.parse(req, (err, fields, files) => {
console.log('err', err) // returns nothing
console.log('files', files) // returns nothing
console.log('fields', fields) // returns nothing
if (err) {
console.log("Error parsing the files");
console.log(err);
return res.status(400).json({
message: "There was an error parsing the files",
status: "Fail",
error: err
})
}
for (let file in files) {
try {
if (files[file]) {
let oldPath = files[file]['path']
let rawData = fs.readFileSync(oldPath)
const month = parseInt(d.getMonth() + 1) < 10 ? '0' + parseInt(d.getMonth() + 1) : parseInt(d.getMonth() + 1)
let today = `${d.getFullYear()}_${month}_${d.getDate()}`
let folderPath = __basedir + `\\media\\uploads\\storage\\${today}\\`;
// folderPath = ..\dashboard-v2.0\server\\media\uploads\storage\\2021_06_18\\
if (!fs.existsSync(folderPath)) {
fs.mkdirSync(folderPath, {
recursive: true
});
}
// newPath =..\dashboard-v2.0\server\\media\uploads\storage\\2021_06_18\\WIN.jpg
let newPath = folderPath + files[file]['name']
let databasePath = `storage/${today}/${files[file]['name']}`;
let filename = files[file]['name'] // example_files.zip
if (fs.existsSync(newPath)){
// if file is existed then add Date.now()
let time = Date.now()
let filenameSplit = filename.split('.')
filename = filenameSplit[0] + '_' + time + '.' + filenameSplit[1]
// filename = WIN_1626750408096.jpg
newPath = folderPath + filename
databasePath = `storage/${today}/${filename}`;
}
fs.writeFile(newPath, rawData, async (err) => {
if (err) {
console.log(err);
return res.status(400).send({ "err": err })
}
const userToken = jwt.verify(fields.user, config.TOKEN_SECRET)
const newFiles = {
filename: filename,
user_id: ObjectId(userToken.id),
filepath: databasePath,
added_time: Date.now(),
}
const result = await db.collection("ate_files").insertOne(newFiles)
console.log(`Created with the following id: ${result.insertedId}`)
console.log(`Successfull upload ${newPath}`);
})
}
} catch (err) {
console.log(`Error: ${err}`);
return res.status(409).send({ "error": `${err}` })
}
}
})
return res.status(200).send({ "message": "Successfully uploadded the files" })
}
Your return res.status(200).send({ "message": "Successfully uploadded the files" }) is too soon, it should be in the callback.
It could be problematic on large files since the beginning of the big file would be received and then the client already receives a response which could logically cut the connection in http

Nodejs bluebird promise fails while processing image

//Created a promise for each image size.
var promises = sizes.map(function (size) {
return new Promise(function (resolve, reject) {
var destinationDir = fileUtil.getAbsolutePathOfImage(destinationPath);
fileUtil.createDirectoryIfNotExists(destinationDir);
destinationDir += size.src;
fileUtil.createDirectoryIfNotExists(destinationDir);
//Resize the image.
//console.log('imagefile : ' + JSON.stringify(imageFile));
//console.log('destinationDir: ' + JSON.stringify(destinationDir));
//Called an imageUtil resize method to perform resize.
imageUtil.resize(imageFile, destinationDir, size).then(data => {
var fileName = destinationPath + size.src + '/' + data;
resolve(imageUtil.createImageData(fileName, size.height, size.width));
}).catch(err => {
console.error(err);
return reject(err);
});
});
});
Promise.all(promises)
.then(savedImages => {
console.log('saved Images are: ' + JSON.stringify(savedImages));
return res.status(200).json(savedImages);
}).catch(err => {
console.log('i am here' + JSON.stringify(err.message));
return res.status(400).json(JSON.stringify(err.message));
});
---------------------Resize method of imageutil---------------
var Promise = require('bluebird'),
gm = require('gm'),
path = require('path'),
fs = require('fs');
Promise.promisifyAll(gm.prototype);
module.exports = {
resize(imageFile, destinationPath, size){
if (!imageFile || !destinationPath || !size) {
return;
}
return new Promise(function (resolve, reject) {
// If we just passed callback directly, errors would be fatal
var fileName = fileUtil.getFileName(imageFile);
//console.log('sourceFile : ' + JSON.stringify(imageFile));
//console.log('saveDirectory : ' + JSON.stringify(destinationPath));
//console.log('fileName is :' + fileName);
//Create a write stream.
var writeStream = fs.createWriteStream(destinationPath + '/' + fileName);
//console.log('Saving at location: ' + writeStream.path);
gm(imageFile)
.resize(size.width, size.height, '^')
.gravity('Center')
.crop(size.width, size.height)
.writeAsync(writeStream.path, function (err) {
if (err) {
var error = 'Error while creating image of resolution : ' + size.width + 'x' + size.height + '.';
console.error(JSON.stringify(error));
return reject(new Error(error));
}
});
resolve(fileName);
});
}
};
*It seems like everything went correct and it creates four image file which is corrupted and gave me error later on but request processed succesfully. Console output of my image processing are as follows:
saved Images are: [{"src":"/uploads/300/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"200","width":"300"},{"src":"/uploads/120/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"120","width":"120"},{"src":"/uploads/48/fhjXFLgqq59F91uFK_2h8GiS.jpg","height":"48","width":"48"}]
POST /api/upload/image/ 200 51.790 ms - 241
"Error while creating image of resolution : 120x120."
"Error while creating image of resolution : 48x48."
"Error while creating image of resolution : 300x200."*
Since you are already using promisifyAll, you don't need to (and should not) use the Promise constructor. writeAsync already returns a promise - if you don't pass a callback, as that's the requirement for Bluebird being able to pass the callback itself in the right position.
You should be using
module.exports = {
resize(imageFile, destinationPath, size){
if (!imageFile || !destinationPath || !size) {
return Promise.reject(new Error("missing arguments"));
}
var fileName = fileUtil.getFileName(imageFile);
//console.log('sourceFile : ' + JSON.stringify(imageFile));
//console.log('saveDirectory : ' + JSON.stringify(destinationPath));
//console.log('fileName is :' + fileName);
//Create a write stream.
var writeStream = fs.createWriteStream(destinationPath + '/' + fileName);
//console.log('Saving at location: ' + writeStream.path);
var promise = gm(imageFile)
.resize(size.width, size.height, '^')
.gravity('Center')
.crop(size.width, size.height)
.writeAsync(writeStream.path);
return promise.then(function() {
return filename;
}, function (err) {
var error = 'Error while creating image of resolution : ' + size.width + 'x' + size.height + '.';
console.error(error, err);
throw new Error(error);
});
}
};
Similarly, you shouldn't use the Promise constructor antipattern in the call to resize - it already returns a promise:
var promises = sizes.map(function (size) {
var destinationDir = fileUtil.getAbsolutePathOfImage(destinationPath);
fileUtil.createDirectoryIfNotExists(destinationDir);
destinationDir += size.src;
fileUtil.createDirectoryIfNotExists(destinationDir);
//Resize the image.
//console.log('imagefile : ' + JSON.stringify(imageFile));
//console.log('destinationDir: ' + JSON.stringify(destinationDir));
return imageUtil.resize(imageFile, destinationDir, size)
//^^^^^^
.then(data => {
var fileName = destinationPath + size.src + '/' + data;
return imageUtil.createImageData(fileName, size.height, size.width));
}, err => {
console.error(err);
throw err;
});
});

local variable is not changing

I'm using node.js for converting text files to CSV. It works for one file, but when i try process more files fileDestination variable doesn't change. Why? Input files like this: r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
I got following console output:
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_1_5X1x9_05_train2.res
/*
* lee archivos *.dat y los convierte *.csv
*/
const fs = require('fs');
const inputDir = './';
const outputDir = './';
function readFiles(inputDir, onError) {
fs.readdir(inputDir, function(err, filenames) {
if (err) {
onError(err);
return;
}
filenames.forEach(function(inputFile) {
// first we arre looking for "right" file name
if (inputFile.search(/res/) != -1) {
console.log('Starting processing ' + inputFile);
convert2csv(inputFile, function(error) {
throw err;
});
}
});
});
}
function convert2csv(filename, onError) {
arrayFromFilename = filename.split('_');
epoca = arrayFromFilename[4];
trainORval = arrayFromFilename[7].replace('2.res', '');
console.log("from convert " + filename + " " + epoca);
fs.readFile(inputDir + filename, 'utf-8', function(err, content) {
if (err) {
onError(err);
return;
}
content = content.replace(/^[^0].*\n/mg, '');
arr = content.split('\n');
pares = arr.filter(function(d, i) {
return i % 2 == 1;
});
content = pares.join('\n');
content = content.replace(/(^[\d.]*) ([\d.]*)/gm, '$1,$2');
fileDestination = outputDir + epoca + '_' + trainORval + '.csv';
console.log("filedestination :" + fileDestination);
fs.writeFile(fileDestination, 'y,x\n', function(err) {
if (err) {
return console.error(err);
}
fs.appendFile(fileDestination, content, function(err) {
if (err) {
return console.error(err);
}
console.log(fileDestination + " has been written successufully!", filename);
});
});
});
}

Node: Trying to rename every file in a directory skips over some

I am trying to go through a directory (and all it's subdirectories) and rename (move) every image if it is a .jpg file. I think my code is correct. I am using asynchronous renaming and an incrementing counter to increase the filenames. For some reason though, in my tests with four or so subfolders only half of the images get renamed. I am not sure why this is, as I am only incrementing when the callback files (once the image has successfully been renamed) and checking the console lists every file that rename is being called on.
var fs = require('fs');
var filePrefix = 'new_Images';
var folderName = 'changed';
var counter = 0;
function examine (path) {
fs.readdir(path, function (err, files) {
files.forEach(function (file) {
var oldPath = path + file;
fs.stat(oldPath, function (err, stats) {
if (stats.isDirectory()) {
examine(oldPath + '/');
} else {
var suffix = oldPath.substr(oldPath.length - 3);
if (suffix === 'jpg') {
fs.rename(oldPath, './' + folderName + '/' + filePrefix + counter + '.jpg', function (e) {
if (e) throw new Error('something bad happened');
console.log('Renamed a .jpg file');
console.log(oldPath);
console.log(counter);
counter++;
});
} else if (suffix === 'png') {
fs.rename(oldPath, './' + folderName + './' + filePrefix + counter + '.png', function (e) {
if (e) throw new Error('something bad happened');
console.log('Renamed a .png file');
console.log(oldPath);
console.log(counter);
counter++;
});
}
}
});
});
});
}
examine('./');
This code will log that every file was renamed, and all of the files will be removed, but only half of them are actually moved (renamed) to the new folder
I'm guessing that the files are being over written.
Since asynchronous calls are made while renaming you should increment counter before making asynchronous call rather than after successful renaming. Since there is no order in asynchronous calls, chances are half of the calls are made before any successful rename operation with the same name.
You can verify this by logging the file name:
if (suffix === 'jpg') {
console.log('renaming with file name ' + filePrefix + counter + '.jpg');
fs.rename(oldPath, './' + folderName + '/' + filePrefix + counter + '.jpg', function (e) {
...
});
} else if (suffix === 'png') {
console.log('renaming with file name ' + filePrefix + counter + '.png');
fs.rename(oldPath, './' + folderName + './' + filePrefix + counter + '.png', function (e) {
...
});
}
This program fulfills all requirements
var fs = require('fs'),
path = require('path');
var oldPath = 'G:\\oldPath\\oldPath';
var newPath = 'G:\\newPath\\newPath\\';
var ext = 'jpg';
var newPrefix = 'newPrefix';
var myFun = function(mypath) {
fs.readdir(mypath, function(error, files) {
if (error)
console.log('error ' + error.code + ' : ' + error.message);
else {
files.map(function(file) {
return path.join(mypath, file)
}).filter(function(file) {
if(fs.statSync(file).isFile())
return file;
else
return myFun(file);
}).forEach(function(file){
var path_file = file.split('\\');
var extension = path_file[path_file.length - 1].split('.');
if (extension[1] === ext) {
var source = fs.createReadStream(file);
var target = fs.createWriteStream(newPath + newPrefix + path_file[path_file.length - 1]);
source.pipe(target);
fs.unlink(file);
}
})
}
})
}

node.js move bunch of files

i have made a form for file upload and i have set the multiple option, so im trying to upload a bunch of files and then move them according to the album name that the client have set,
here is what iv done:
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
var file_name = null;
switch(req.files.img_file.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
default:
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ לא תקין'
});
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
else
{
_.each(req.files.img_file,function(val,index){
console.log(val.path + " " + index);
//gives the file path so i can read it
fs.readFile(val.path, function (err, data) {
if (err){
console.log("fs " + err);
}
//so until here everything works fine, the files are uploaded to the "/uploads" directory, now im trying to move them to the correct album, the destiation is : public/img/albums/:album_name/:all_images here
mv(val.path, './public/img/albums/' + album_name + '/' + val.path, function(err) {
if (err){
console.log("mv " + err);
}
else{
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה'
});
res.redirect('/admin');
}
});
});
});
}
});
the mv module throws an error rename "c:/work/xxx/xx/uploads/val.path.png
Its a file-access error. You X out some of the file name, but look at how that NPM module handles file, and ensure you are naming the files and paths properly. Then it should work out fine.
i have used read and write stream and deleted the "mv" module
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
if (req.files.img_file.length > 1)
{
var Counter = 0;
_.each(req.files.img_file,function(val,index){
var file_name = null;
switch(val.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
var source = fs.createReadStream(val.path);
var dest = fs.createWriteStream('./public/img/albums/' + album_name + '/' + val.name);
source.pipe(dest);
source.on('end', function() {
console.log('end...');
Counter++;
console.log(Counter);
console.log(req.files.img_file.length);
if (Counter == req.files.img_file.length){
res.redirect('/admin');
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה',
albums: albums
}); //eo res render
}
});
source.on('error', function(err) { console.log('error'); });
});// eo mkdir
}); // eo _each
}

Resources