local variable is not changing - node.js

I'm using node.js for converting text files to CSV. It works for one file, but when i try process more files fileDestination variable doesn't change. Why? Input files like this: r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
I got following console output:
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_10_5X1x9_05_train2.res
./1_train.csv has been written successufully! r10_1C_BP1-11_e41-81_1_5X1x9_05_train2.res
/*
* lee archivos *.dat y los convierte *.csv
*/
const fs = require('fs');
const inputDir = './';
const outputDir = './';
function readFiles(inputDir, onError) {
fs.readdir(inputDir, function(err, filenames) {
if (err) {
onError(err);
return;
}
filenames.forEach(function(inputFile) {
// first we arre looking for "right" file name
if (inputFile.search(/res/) != -1) {
console.log('Starting processing ' + inputFile);
convert2csv(inputFile, function(error) {
throw err;
});
}
});
});
}
function convert2csv(filename, onError) {
arrayFromFilename = filename.split('_');
epoca = arrayFromFilename[4];
trainORval = arrayFromFilename[7].replace('2.res', '');
console.log("from convert " + filename + " " + epoca);
fs.readFile(inputDir + filename, 'utf-8', function(err, content) {
if (err) {
onError(err);
return;
}
content = content.replace(/^[^0].*\n/mg, '');
arr = content.split('\n');
pares = arr.filter(function(d, i) {
return i % 2 == 1;
});
content = pares.join('\n');
content = content.replace(/(^[\d.]*) ([\d.]*)/gm, '$1,$2');
fileDestination = outputDir + epoca + '_' + trainORval + '.csv';
console.log("filedestination :" + fileDestination);
fs.writeFile(fileDestination, 'y,x\n', function(err) {
if (err) {
return console.error(err);
}
fs.appendFile(fileDestination, content, function(err) {
if (err) {
return console.error(err);
}
console.log(fileDestination + " has been written successufully!", filename);
});
});
});
}

Related

How to copy files to folder with same name even the files are different format ( PDF, EXCEL, PPT, WORD ) ? ( Using Node )

I am trying to copy multiple files even the same name with a different format. ( file.ppt , file.pdf, etc.. )
So far I can able to copy to a folder in any one format file ( if the files are the same name with different format )
For Example: If I have name1.ppt, name1.doc, name1.pdf. I need to copy all the files into the respective folder. But now anyone of a format file from the same name is copying to the folder. I need to copy all the files from the same name.
Please check my below code,
router.get('/segregate', async(req, res, next) => {
removeDir('C:\\IND\\Processfile') // I just resetting folder.
createDir();
try {
// console.log(res)
return res.status(200).json({ status: 200, data: 'success' });
} catch (err) {
next(err);
}
})
const removeDir = function(path) {
if (fs.existsSync(path)) {
const files = fs.readdirSync(path)
if (files.length > 0) {
files.forEach(function(filename) {
if (fs.statSync(path + "/" + filename).isDirectory()) {
removeDir(path + "/" + filename)
} else {
fs.unlinkSync(path + "/" + filename)
}
})
fs.rmdirSync(path)
} else {
fs.rmdirSync(path)
}
} else {
console.log("Directory path not found.")
}
}
async function createDir() {
console.log("Yes ssssssss");
// let newList = [];
let clearDirFlag = false;
let masterData = await MasterModel.find().lean();
// This is the folder where i need to copy the segregated files
fs.mkdir("C:\\IND\\Processfile", (err) => {
if (err) {
return;
}
clearDirFlag = true;
// console.log("clearDirFlagclearDirFlagclearDirFlagclearDirFlag ", clearDirFlag)
});
if(masterData.length!=0){
Object.keys(masterData).map( value => {
let newList = []
if (!masterData[value]['CLIENTCODE'].toString().match(/^[0-9a-z]+$/)){
// console.log("Yes ", data)
let answer = masterData[value]['CLIENTCODE'].toString().replace(/[^0-9]/g, '');
// console.log("answer ", answer);
newList.push(
{
"clientcode": answer,
"interm" : masterData[value]['INTERMEDIARYNAME']
}
)
} else{
// console.log("No ")
newList.push(
{
"clientcode": masterData[value]['CLIENTCODE'],
"interm" : masterData[value]['INTERMEDIARYNAME']
}
)
}
let filename;
let interm;
let boo = false;
let name = newList[0]['clientcode'];
var to_zip = fs.readdirSync("C:\\IND\\BGLHYD");
to_zip.forEach((name1) => {
if (name1.toString().includes(name)) {
// console.log(name1);
filename = name1;
interm = newList[0]['interm'];
boo = true;
}
});
if(boo == true) {
const pathToFile = "C:\\IND\\BGLHYD\\"+ filename;
// console.log("hello123", pathToFile);
fs.mkdir("C:\\IND\\Processfile\\" + interm, (err) => {
if (err) {
return;
}
});
const pathToNewDestination = "C:\\IND\\Processfile\\" + interm +"\\"+ filename;
let readStream = fs.createReadStream(pathToFile);
readStream.once('error', (err) => {
console.log(err);
});
readStream.once('end', () => {
console.log('done copying');
});
readStream.pipe(fs.createWriteStream(pathToNewDestination));
}
})
}
}
Here is the place i doing the copy functionality
if(boo == true) {
const pathToFile = "C:\\IND\\BGLHYD\\"+ filename;
// console.log("hello123", pathToFile);
fs.mkdir("C:\\IND\\Processfile\\" + interm, (err) => {
if (err) {
return;
}
});
const pathToNewDestination = "C:\\IND\\Processfile\\" + interm +"\\"+ filename;
let readStream = fs.createReadStream(pathToFile);
readStream.once('error', (err) => {
console.log(err);
});
readStream.once('end', () => {
console.log('done copying');
});
readStream.pipe(fs.createWriteStream(pathToNewDestination));
}
Please help me to solve this issue.

Create and download text files- Node.js & React.Js

As part of my project, I have to create text files which have to be downloaded as a ".txt".
I am using Node.js and React JavaScript, and I have already tried using the Node.js "fs.writeFile", but the browser doesn't recognize the download, the file name is always being called as the folder name and the file is always empty although the variable is a string and not empty.
I'm calling from the client to this function:
app.post("/downloadnetworks", async (req, res) => {
let selectedApps=req.body.selectedApps;
let arr=await sqlFunctions.createByIds(selectedApps);
res.send();
module.exports.createByIds = (productsArray) => {
return new Promise(function(resolve, reject) {
var bulkedString = '';
var product;
for (let obj of productsArray) {
let query = "select * from...........";
con.query(query, function(err, result, fields) {
if (err) throw err;
let stringifiedJson = JSON.stringify(result)
let parsedJson = JSON.parse(stringifiedJson)
The DB data is being added into the variable 'stringifiedJson', and it continues from here:
let parsedJson = JSON.parse(stringifiedJson) //has all the data from the DB
for (let network of parsedJson) {
if (network.certification_Id) {
bulkedString += network.domain_Name + ", " + network.publisher_Id + ", " + network.relationship + ", " + network.certification_Id;
} else {
bulkedString += network.domain_Name + ", " + network.publisher_Id + ", " +
network.relationship;
}
bulkedString += "\n";
product = network.product;
}
})
fs.writeFile('C:\Work\App ads.txt\App-Ads Files\'' + product + '.txt', bulkedString, 'utf8', (err) => {
if (err) throw err;
console.log('The file has been saved!');
});
}
resolve(bulkedString)
})
}

NodeJS itself keeps file EBUSY on Windows?

I created a simple function to process uploaded files. I'm using multer to process the multipart data into files. Then I use the code below to move the files around, and return data so my webpage knows how to display the images.
It seems that somehow NodeJS keeps the files open itself. I also created a function to remove the files, but this will give me an EBUSY error. If I try to remove through Windows, it says that NodeJS has te file locked. When I restart the NodeJS process and then re-request the delete URL, the file is removed correctly.
Is there some way I can force NodeJS to close the file resources? Or is there some other error in my script that I am missing?
I updated node to version 12.4.0 but this didn't help either.
Processing the uploads:
exports.handleFormNotes = async(req, res, next) => {
try {
const configVariables = req.app.get('configVariables');
const uploadSuffix = req.body.uploadFolderSuffix || '';
console.log('upload suffix', uploadSuffix);
if (!req.files.length) {
return;
}
const uploadedFiles = Array();
var destPath = configVariables['FormNotesUploadDirectory'];
if (uploadSuffix !== '')
destPath = destPath + '/' + uploadSuffix;
destPath = path.resolve(destPath);
// mkdirSync returns undefined, so run that first and see if the directory exists second.
if (!fs.mkdirSync(destPath, { recursive: true }) && !fs.existsSync(destPath)) {
console.log(destPath, 'does not exist!');
req.alertHandler.addAlert('Pad om afbeelding op te slaan is niet bereikbaar: ' + destPath, 'danger');
res.render('error');
return;
}
var baseUrlPath = configVariables['FormNotesUploadDocumentRoot'];
if (uploadSuffix != null) {
baseUrlPath = baseUrlPath + '/' + uploadSuffix;
}
for(const uploadedFile of req.files) {
let now = new Date();
let destFilename = getDateTime() + "_" + uploadedFile.originalname;
let destFilenameThumb = 'thumb_' + destFilename;
var fullDestination = path.resolve(destPath + '/' + destFilename);
var fullDestinationThumb = path.resolve(destPath + '/' + destFilenameThumb);
console.log('Copy src:', uploadedFile.path, fullDestination);
fs.copyFileSync(uploadedFile.path, fullDestination);
var unlinkResult = fs.unlinkSync(uploadedFile.path);
console.log('Unlink "' + uploadedFile.path + '", result after upload:', unlinkResult);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);
console.log('new file info thumb:', newFileInfo);
uploadedFiles.push({
'fullImg': baseUrlPath + '/' + destFilename,
'thumbImg' : baseUrlPath + '/' + destFilenameThumb,
'original': uploadedFile.originalname
});
}
// Push to backend
const data = {
files: [...uploadedFiles],
uploadSuffix: uploadSuffix
};
// Normally retVal should be the return data from OI. If anything goes wrong, retVal = 'error'
this.saveAttachment(req, res, data);
return res.send(data);
}
catch (err) {
console.log('Error handling from notes:', err);
req.alertHandler.addAlert('Error handling form notes: ' + err);
return 'error';
}
}
Removing the uploads:
exports.rmFormNote = async(req, res, data) => {
let retVal;
try {
const configVariables = req.app.get('configVariables');
const httpPath = req.query.img;
console.log('http path:', httpPath);
// Strip off the document root, but check if they are the same first
const firstPart = httpPath.substring(0, configVariables['FormNotesUploadDocumentRoot'].length);
console.log('same?', firstPart, configVariables['FormNotesUploadDocumentRoot']);
var relPath = httpPath;
if (firstPart == configVariables['FormNotesUploadDocumentRoot']) {
relPath = httpPath.substring(configVariables['FormNotesUploadDocumentRoot'].length + 1);
}
var parts = relPath.split('/');
parts[parts.length-1] = 'thumb_' + parts[parts.length-1];
var thumbPath = parts.join('/');
thumbPath = path.resolve(configVariables['FormNotesUploadDirectory'] + '/' + thumbPath);
console.log('thumbpath: ', thumbPath);
var fullPath = configVariables['FormNotesUploadDirectory'] + '/' + relPath;
var dest = path.resolve(fullPath);
console.log('dest: ', dest);
if (!fs.existsSync(dest))
throw "File not found";
fs.unlink(dest, (err) => {
if (err) throw err;
console.log('File deleted');
});
retVal = { result: true };
}
catch(err) {
console.log('Ohnoo', err);
retVal = { result: false, msg: err };
}
return res.send(retVal);
}
Turns out the thumbnail creator sharp was the problem, as stated in this github issue.
I just had to disable the cache, like so:
sharp.cache(false);
var newFileInfo = await sharp(destPath + '/' + destFilename)
.resize({ width: 120 })
.toFile(fullDestinationThumb);

Node.js multer simulanteous uploads

Trying to get working a system for uploading documents from tablets.
Using multer and express 4.
While using 2 or 3 devices simulanteously everything is okay, but when adding more devices, filename troubles are taking place. Multer provides the same filename for all of the uploads (watching console.log(storedFile + " " + idPatient);). Maybe something is done wrong, but can't figure what exactly.
app.post("/api/Upload", jsonParser, function(req, res) {
var saveFilename = "",
savePath = "",
savePathForSql = "",
fileToSave = "";
var idDocument = 0,
idPatient = 0,
idDoctor = 0;
var uploadRequest = "";
var f = "queryLog.txt";
async.series([
function(callback) {
upload = multer({ storage: Storage }).single("imgUploader");
upload(req, res, function(err) {
if (err) {
console.log(err);
return res.end("Something went wrong!");
}
fileToSave = storedFile;
uploadRequest = req;
idDocument = JSON.parse(req.body.json)['id_doc_type'];
idPatient = JSON.parse(req.body.json)['id_patient'];
idLogin = JSON.parse(req.body.json)['id_login'];
mv("Images" + separator + fileToSave, idPatient + ".jpg", function(err) {
if (err) {
console.log(err);
} else {
return res.end("File uploaded sucessfully! ");
}
});
console.log(storedFile + " " + idPatient);
callback();
});
},
function(callback) {
var request = new sql.Request()
var q1 = "exec storedproc";
request.query(q1, (err, result) => {
if (err) return callback(err);
console.log(result.recordset[0]);
savePath = result.recordset[0]['path'];
savePathForSql = savePath;
if (os != 'linux') {
savePath = savePath.replaceAll("/", "\\");
}
if (!fs.existsSync(f)) {
fs.writeFileSync(f, q1 + "\r\n", 'utf-8');
} else {
fs.appendFileSync(f, q1 + "\r\n", 'utf-8');
}
saveFilename = result.recordset[0]['filename'];
console.log(savePath + "/" + saveFilename);
callback();
})
},
function(callback) {
mkdirp(basePath + savePath, function(err) {
mv("Images" + separator + idPatient + ".jpg", basePath + savePath + separator + saveFilename, function(err) {
if (err) {
console.log(err);
console.log("Move failed: Images/" + idPatient + ".jpg" + " to " + basePath + savePath + separator + saveFilename);
} else {
console.log('Move complete. Images/' + idPatient + ".jpg" + " to " + basePath + savePath + separator + saveFilename);
return res.end("File uploaded sucessfully! ");
}
});
});
callback();
}
], function(err) {
var request2 = new sql.Request()
var q2 = "exec storedproc";
request2.query(q2, (err2, result2) => {
if (err2) return callback(err2);
if (err2) {
console.log(err2);
}
});
});
});
Multer config is:
Storage = multer.diskStorage({
destination: function(req, file, callback) {
callback(null, "./Images");
},
filename: function(req, file, callback) {
storedFile = file.fieldname + "_" + Date.now() + "_" + file.originalname + randomInteger(99999) + ".jpg"
callback(null, storedFile);
}
});
Multer adds file information to the request object.
req.file
Retrieve the uploaded image path from:
req.file.path
Your globally stored variable storedFile is what's tripping you. Update your code to use the image path string:
mv( req.file.path , idPatient + ".jpg", function(err) {

node.js move bunch of files

i have made a form for file upload and i have set the multiple option, so im trying to upload a bunch of files and then move them according to the album name that the client have set,
here is what iv done:
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
var file_name = null;
switch(req.files.img_file.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
default:
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ לא תקין'
});
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
else
{
_.each(req.files.img_file,function(val,index){
console.log(val.path + " " + index);
//gives the file path so i can read it
fs.readFile(val.path, function (err, data) {
if (err){
console.log("fs " + err);
}
//so until here everything works fine, the files are uploaded to the "/uploads" directory, now im trying to move them to the correct album, the destiation is : public/img/albums/:album_name/:all_images here
mv(val.path, './public/img/albums/' + album_name + '/' + val.path, function(err) {
if (err){
console.log("mv " + err);
}
else{
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה'
});
res.redirect('/admin');
}
});
});
});
}
});
the mv module throws an error rename "c:/work/xxx/xx/uploads/val.path.png
Its a file-access error. You X out some of the file name, but look at how that NPM module handles file, and ensure you are naming the files and paths properly. Then it should work out fine.
i have used read and write stream and deleted the "mv" module
if (req.body && req.body.album){
var album_name = req.body.album;
}
else{
//need to change to time instead of random album
var album_name = 'unknown_album-' + (parseInt(Math.random() * 5) + 1);
}
//File name
if (req.files.img_file.length > 1)
{
var Counter = 0;
_.each(req.files.img_file,function(val,index){
var file_name = null;
switch(val.type){
case 'image/png':
file_name = new Date().getTime() + '.png';
break;
case 'image/jpeg':
file_name = new Date().getTime() + '.jpeg';
break;
}
mkdirp('./public/img/albums/' + album_name, function (err) {
if (err)
console.error(err);
var source = fs.createReadStream(val.path);
var dest = fs.createWriteStream('./public/img/albums/' + album_name + '/' + val.name);
source.pipe(dest);
source.on('end', function() {
console.log('end...');
Counter++;
console.log(Counter);
console.log(req.files.img_file.length);
if (Counter == req.files.img_file.length){
res.redirect('/admin');
res.render('admin/panel', {
title: 'אדמין',
message: 'קובץ עלה בהצלחה',
albums: albums
}); //eo res render
}
});
source.on('error', function(err) { console.log('error'); });
});// eo mkdir
}); // eo _each
}

Resources