Function call inside loop gets called after the loop ends - node.js

So I use this API that helps me turn a .docx file into a .pdf. I placed the code that converts the file into a function. :
function conv(){
convertapi.convert('pdf', { File: final_path })
.then(function(result) {
// get converted file url
console.log("Converted file url: " + result.file.url);
finp = path + file_name.slice(0, file_name.length - 5) + ".pdf";
console.log(finp);
// save to file
return result.file.save(finp);
})
.then(function(file) {
console.log("File saved: " + file);
process.exit(1);
})
.catch(function(e) {
console.log("numele si/sau extensia fisierului sunt gresite");
process.exit(1);
});
}
The code above works only for one file at a time. I made a loop that goes through every file (.docx) in my folder and save its name into an array. I go through every item of the array and call the function :
for(var j = 0; j<=i ; j++){
file_name = toate_nume[j];
final_path = path + file_name;
conv();
}
The file names are stored correctly, but when I run my project, the function is called after the loop itself ends ( is called the correct number of times for each and every file). So if I have 2 files : test1.docx and test2.docx the output shows me that the conv() is called 2 times for the test2.docx, instead of one time for each file. What should I do?

The reason might be this:
The API is slow so your program is executing the loop faster than the API the can handle the requests. So what ends up happening is that you have modified the final_path variable twice before convertapi gets called, and then it gets called twice with the same final_path. Try to modify your conv function so that it accepts a parameter, e.g. path and uses that. Then call conv with the current final_path parameter:
conv(final_path)
And:
function conv(path) {
convertapi.convert('pdf', { File: path })
...

So you are calling n Promise in a serial. And you want to wait for the end?
You can use Promise. all
const toate_nume = ['fileName1', 'fileName2'];
const spawn = toate_nume.map(x => {
const final_path = path + x;
return conv(final_path);
});
Promise.all(spawn).then(results => {
console.log('All operation done successfully %o', results);
});
or use await:
const results = await Promise.all(spawn);
the results is an array, an entry for each call.
NOTE** I pass the path as an argument instead of a global var

Related

How to wait for the response of all async fs.rename without blocking the process?

I built an Angular/Node app that renames files in network folders. The number of files it renames are between 300 to 500. I use await so I get notified when renaming is done. It takes 8-10minutes per run and it can't rename simultaneously since I am using await.
I need to pass the number of renamed files and I need to show the user that the renaming is already complete. If I don't use async/await, how can my angular front-end know that the renaming is completed?
My full code is in here: https://github.com/ericute/renamer
Here's where I'm having a trouble with:
await walk(folderPath, function(err, results) {
if (err) throw err;
results.forEach(file => {
if (fs.lstatSync(file).isFile) {
fileCounter++;
}
let fileBasename = path.basename(file);
let filePath = path.dirname(file);
if (!filesForRenaming[path.basename(file)]) {
//In a javascript forEach loop,
//return is the equivalent of continue
//https://stackoverflow.com/questions/31399411/go-to-next-iteration-in-javascript-foreach-loop
return;
}
let description = filesForRenaming[path.basename(file)].description;
// Process instances where the absolute file name exceeds 255 characters.
let tempNewName = path.resolve(filePath, description + "_" + fileBasename);
let tempNewNameLength = tempNewName.length;
let newName = '';
if (tempNewNameLength > 255) {
let excess = 254 - tempNewNameLength;
if (description.length > Math.abs(excess)) {
description = description.substring(0, (description.length - Math.abs(excess)));
}
newName = path.resolve(filePath, description + "_" + fileBasename);
} else {
newName = tempNewName;
}
renamedFiles++;
// Actual File Renaming
fs.renameSync(file, newName, (err) => {
if (err) {
errList.push(err);
}
renamedFiles++;
});
});
if (Object.keys(errList).length > 0) {
res.send({"status":"error", "errors": errList});
} else {
res.send({
"status":"success",
"filesFoundInDocData": Object.keys(filesForRenaming).length,
"filesFound": fileCounter,
"renamedFiles": renamedFiles,
"startDate": startDate
});
}
});
If you're using any sync methods you're basically blocking the event loop. You must change the whole structure of your code and start using promises everywhere. You should be able to create another service in angular that checks if the renaming process is completed using timeInterval and GET requests (the most easy way). For example, you could set angular to fetch data from "/isRenameCompleted" and alert the user if the result is true or something. To get real-time results you must switch to socket-io. A quickly solution for 1 client (cause you need to store unique IDs for each request and fetch promises accordingly) could be this one:
1: Create two global variables on top of your code
var filesStatus="waiting"
var pendingFiles=[]
2: Inside your renaming logic route push every file to the promise array using a for loop and start waiting asynchronously for the renaming process to finish
pendingFiles.push(fsPromises.rename(oldName,newName))
Promise.all(pendingFiles)
.then(values => {
filesStatus = "done"
})
.catch(error => {
filesStatus = "error"
});
filesStatus="pending"
3: Now add a new route /isRenameCompleted that will have a report logic like the following
router.get('/isRenameCompleted', (req, res, next) => {
if (filesStatus==="pending"){
res.end("please wait")
} else if (filesStatus==="done"){
res.end("done! your files renamed")
}
}

How to use promises correctly with multiple piped writestreams

var promisePipe = require("promisepipe");
var fs = require("fs");
var crypt = require("crypto");
var // ....
files = ['/mnt/Storage/test.txt', '/mnt/Storage/test2.txt', '/mnt/Storage/test3.txt']
var promises = files.map(function(file_enc) {
return new Promise(function(resolve, reject) {
var file_out = file_enc + '.locked';
promisePipe(
fs.createReadStream(file_enc),
crypt.createCipheriv(alg, genhashsub, iv),
fs.createWriteStream(file_out),
).then(function(streams){
console.log('File written: ' + file_out);
// Promise.resolve(file_out); // tried but doesnt seem to do anything
}, function(err) {
if(err.message.substring(0, 7) === 'EACCES:') {
console.log('Error (file ' + file_out + '): Insufficient rights on file or folder');
} else {
console.log('Error (file ' + file_out + '): ' + err);
}
// Promise.reject(new Error(err)); // tried but doesnt seem to do anything
});
})
});
Promise.all(promises).then(final_function(argument));
I'm trying to encrypt files contained in an array named files.
For the sake of simplicity I added them manually in this example.
What I want to happen:
Create promises array to call with promises.all on completion
Iterate through the array
Create promise for this IO operation
Read file \
Encrypt file -- all done using streams, due to large files (+3GB)
Write file /
On finish write, resolve promise for this IO operation
Run finishing script once all promises have resolved (or rejected)
What happens:
Encryption of first file starts
.then(final_function(argument)) is called
Encryption of first file ends
The files all get encrypted correctly and they can be decrypted afterwards.
Also, errors are displayed, as well as the write confirmations.
I've searched both Stack as well as Google and I have found some similar questions (with answers). But they don't help because many are outdated. They work, until I rewrite them into promises, and then I'm back to where I started.
I could also place 8 different ways to achieve this job, using npm modules or vanilla code, but all of them are also failing in one way or another.
If you already have a promise at your disposal (and promisepipe appears to create a promise), then you generally should not use new Promise(). It looks like your main problem is that you are creating promises that you never resolve.
The other problem is that you are calling final_function in the last line instead of passing a function that will call final_function.
I suggest giving this a try:
var promises = files.map(function(file_enc) {
var file_out = file_enc + '.locked';
return promisePipe(
fs.createReadStream(file_enc),
crypt.createCipheriv(alg, genhashsub, iv),
fs.createWriteStream(file_out),
).then(function(streams){
console.log('File written: ' + file_out);
return file_out;
}, function(err) {
if(err.message.substring(0, 7) === 'EACCES:') {
console.log('Error (file ' + file_out + '): Insufficient rights on file or folder');
} else {
console.log('Error (file ' + file_out + '): ' + err);
}
throw new Error(err);
});
});
Promise.all(promises).then(() => final_function(argument));
an analog of short , file-based process wrapped in a promise all . You could do your encrypt in the 'encrypt' which is wrapped in file handler. encrypt() returns a promise.
segments passes your array of files needing work.
var filHndlr = function(segment){
var uri = segment.uri;
var path = '/tmp/' + uri;
return that.getFile(path)
.then (datatss => {
return that.encrypt(uri, datatss);
});
}
...
Promise.all(segments.map(filHndlr))
.then(resp => { ... });

NodeJS - fs.stat() is ignored in a FOR-loop

I'm trying to loop over a bunch of directories and then try if a file inside that directory exists with NodeJS and fs.stat().
I've got a simple for-loop to loop over the directories and in it the fs.stat() call to check, if "project.xml" inside that particular directory exists. My code looks like this:
for (var i = 0, length = prDirs.length; i < length; i++) {
var path = Config["ProjectDirectory"] + "/" + prDirs[i];
console.log("PATH=" + path);
fs.stat(path + "/project.xml", function (err, stat) {
if (err == null) {
console.log(" => PATH=" + path);
}
})
}
NodeJS loops correctly over the directory, the console.log() outputs all the directories correctly, but the code inside the if inside fs.stat() is not called and runs only once at the end of the loop. My console shows this:
PATH=(...)/PHP
PATH=(...)/Electron
PATH=(...)/testapp
PATH=(...)/Vala
=> PATH=(...)/Vala/project.xml
But the project.xml I'm looking for is in testapp/ not in Vala/ but Vala/ is the last entry in prDirs.
The code above is my latest attempt, I've tried plenty of other variations and one (I appended an else to the if inside fs.stat()) showed me, that fs.stat() actually gets invoked, but only the code inside the if is not running and the code in the else I appended once was running.
Thanks in advance!
fs.stat is an asynchronous i/o function, so its callback will be called only after the main thread is idle, or in your case, only after the for loop is done. Instead of a for loop, I suggest iterating the folder in an asynchronous matter. You can use async.each, async.eachSeries, or implement it yourself.
As #Gil Z mention fs.stat in async. I would suggest to use promises if you want to keep for each loop and make code looks sync.
Here is the example. Works on node v 6.9
"use strict";
const fs = require('fs');
let paths = ["tasks", "aaa", "bbbb"];
//method to get stat data using promises
function checkFileExists(path) {
return new Promise((resolve, reject) => {
fs.stat(path + "/project.xml", (err, res) => {
resolve(err ? "Not found in " + path : " => PATH=" + path);
});
});
}
//create promise array with each directory
let promiseArr = [];
paths.forEach(pathPart => {
let path = process.cwd() + "/" + pathPart;
promiseArr.push(checkFileExists(path));
});
//run all promises and collect results
Promise.all(promiseArr)
.then(reslut => {
console.log(reslut);
})
.catch(e => console.log("Error in promises"));
The above code will log this
[ ' => PATH=/Users/mykolaborysiuk/Sites/circlemedia/syndic-apiManager/tasks',
'Not found in /Users/mykolaborysiuk/Sites/circlemedia/syndic-apiManager/aaa',
'Not found in /Users/mykolaborysiuk/Sites/circlemedia/syndic-apiManager/bbbb' ]
Hope this helps.

Why is the following readFileSync function returning undefined?

I want to read files from a folder and then return the result plus some xhtml:
#! /usr/bin/env node
var fs = require('fs'),
files = fs.readdirSync(__dirname + '/files/')
var manifest = function() {
files.forEach((file) => {
return `<item href="${file}.html" id="html30" media-type="application/xhtml+xml"/>`
})
}
console.log(manifest())
console.log(manifest()) returnes undefined, though. Not sure why, since console.log(contents) returns the result:
foo 1
foo 2
foo 3
Returning a string from forEach isn't going to assign anything since ForEach does not produce any side effects. From mdn
forEach() executes the callback function once for each array element; unlike map() or reduce() it always returns the value undefined and is not chainable. The typical use case is to execute side effects at the end of a chain.
You'll need to reduce over your files (into a string) or mutate a variable.
var manifest = function() {
return files.reduce((items, file) => {
let contents = fs.readFileSync(__dirname + '/files/' + file, 'utf8')
return items + `<item href="${file}.html" id="html30" media-type="application/xhtml+xml"/>`
}, '');
}

Add a mongo request into a file and archive this file

I'm having some troubles while trying to use streams with a MongoDB request. I want to :
Get the results from a collection
Put this results into a file
Put this file into a CSV
I'm using the archiver package for the file compression. The file contains csv formatted values, so for each row I have to parse them in the CSV format.
My function take a res (output) parameters, which means that I can send the result to a client directly. For the moment, I can put this results into a file without streams. I think I'll get memory troubles for a large amount of data that's why I want to use streams.
Here is my code (with no stream)
function getCSV(res,query) {
<dbRequest>.toArray(function(err,docs){
var csv = '';
if(docs !== null){
for(var i = 0; i< docs.length; i++){
var line = '';
for(var index in docs[i]){
if(docs[i].hasOwnProperty(index) && (index !== '_id' ) ){
if(line !== '') line+= ',';
line += docs[i][index];
}
}
console.log("line",line);
csv += line += '\r\n';
}
}
}.bind(this));
fileManager.addToFile(csv);
archiver.initialize();
archiver.addToArchive(fileManager.getName());
fileManager.deleteFile();
archiver.sendToClient(res);
};
Once the csv is completed, I had it to a file with a Filemanager Object. The latter one handles file creation and manipulation. The addToArchive method add the file to the current archive, and the sendToClient method send the archive through the output (res parameter is the function).
I'm using Express.js so I call this method with a server request.
Sometimes the file contains data, sometimes it is empty, could you explain me why ?
I'd like to understand how streams works, how could I implement this to my code ?
Regards
I'm not quite sure why you're having issue with the data sometimes showing up, but here is a way to send it with a stream. A couple of points of info before the code:
.stream({transform: someFunction})
takes a stream of documents from the database and runs whatever data manipulation you want on each document as it passes through the stream. I put this function into a closure to make it easier to keep the column headers, as well as allow you to pick and choose which keys from the document to use as columns. This will allow you to use it on different collections.
Here is the function that runs on each document as it passes through:
// this is a closure containing knowledge of the keys you want to use,
// as well as whether or not to add the headers before the current line
function createTransformFunction(keys) {
var hasHeaders = false;
// this is the function that is run on each document
// as it passes through the stream
return function(document) {
var values = [];
var line;
keys.forEach(function(key) {
// explicitly use 'undefined'.
// if using !key, the number 0 would get replaced
if (document[key] !== "undefined") {
values.push(document[key]);
}
else {
values.push("");
}
});
// add the column headers only on the first document
if (!hasHeaders) {
line = keys.join(",") + "\r\n";
line += values.join(",");
hasHeaders = true;
}
else {
// add the line breaks at the beginning of each line
// to avoid having an extra line at the end
line = "\r\n";
line += values.join(",");
}
// return the document to the stream and move on to the next one
return line;
}
}
You pass that function into the transform option for the database stream. Now assuming you have a collection of people with the keys _id, firstName, lastName:
function (req, res) {
// create a transform function with the keys you want to keep
var transformPerson = createTransformFunction(["firstName", "lastName"]);
// Create the mongo read stream that uses your transform function
var readStream = personCollection.find({}).stream({
transform: transformPerson
});
// write stream to file
var localWriteStream = fs.createWriteStream("./localFile.csv");
readStream.pipe(localWriteStream);
// write stream to download
res.setHeader("content-type", "text/csv");
res.setHeader("content-disposition", "attachment; filename=downloadFile.csv");
readStream.pipe(res);
}
If you hit this endpoint, you'll trigger a download in the browser and write a local file. I didn't use archiver because I think it would add a level of complexity and take away from the concept of what's actually happening. The streams are all there, you'd just need to fiddle with it a bit to work it in with archiver.

Resources