I want to read the all (text) files from a specific directory and it's all subdirecoty recursively.. I am able to read the file and append the result to a global variable. but i want to access the variable at the end of all operation. I am trying with promises but i am unable to access it. please help
var file_path = `C:\\Users\\HP\\Desktop\\test_folder`;
const fs = require('fs');
var final_array = [];
let getFolderTree = function(file_path) {
return new Promise(function(resolve, reject) {
fs.readdir(file_path, function(err, folders) {
if (err) {
console.log("error reading folder :: " + err);
} else {
if (folders.length !== 0) {
for (let i = 0; i < folders.length; i++) {
if (folders[i].endsWith("txt")) {
let text_file_path = file_path + `\\` + folders[i];
fs.readFile(text_file_path, function(error_read, data) {
if (error_read) {
console.log("error reading " + error_read);
} else {
return resolve(final_array.push(data));// want to access final_array at the end of all operations
}
});
} else {
let current_path = file_path + `\\` + folders[i];
getFolderTree(current_path);
}
}
}
}
});
});
}
getFolderTree(file_path).then(function() {
console.log(final_array); // this is not working
});
I think i have found the solution but I am still confused about how it works.
I took reference from another code and able to figure out some how.
var fs = require('fs');
var path = require('path');
let root_path = "C:\\Users\\HP\\Desktop\\test_folder";
function getAllDirectoriesPath(current_path) {
var results = [];
return new Promise(function (resolve, reject) {
fs.readdir(current_path, function (erro, sub_dirs) {
if (erro) {
console.log(error);
} else {
let no_of_subdir = sub_dirs.length;
if (!no_of_subdir) {
return resolve(results);
} else {
sub_dirs.forEach(function (dir) {
dir = path.resolve(current_path, dir);
fs.stat(dir, function (err, stat) {
if (stat && stat.isDirectory()) {
getAllDirectoriesPath(dir).then(function (res) {
results = results.concat(res);
if (!--no_of_subdir) {
resolve(results);
}
});
} else {
fs.readFile(dir, function (err, data) {
results.push(data.toString());
if (!--no_of_subdir) {
resolve(results);
}
});
}
});
});
}
}
});
});
}
getAllDirectoriesPath(root_path).then(function (results) {
console.log(results);
});
I am having a problem using async waterfall where I find that after calling the second callback (cbNumPages), the first parameter "pages" is the actual callback for the next function, instead of the last parameter "cbGetFiles" which it should be (as far as I know async waterfall says that last parameter should always be the callback, well in this case is apparently not).
The code is the following:
async.waterfall
([
function(cbNumPages)
{
request({
url: 'any-url',
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if (!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var pagesList = $('ol.aui-nav').children();
if(pagesList.length<1)
{
var numPages = 1;
} else {
var numPages = pagesList.length-2;
}
console.log(numPages);
var pages = new Array(numPages),
total = numPages*20,
iterator = 0;
async.eachSeries(pages, function(page, cb)
{
if(page>1)
{
pages[iterator] = iterator;
}else {
pages[iterator] = iterator*20;
}
iterator++;
cb();
}, function(err){
if(err) cbNumPages(err);
cbNumPages(null, pages);
});
} else {
cbNumPages(err);
}
})
},
function(pages, cbGetFiles)
{
var files = [];
var limitDate = moment().tz('Europe/Madrid').subtract(330,'days').format();
async.eachSeries(pages, function(page, cb)
{
request({
url: 'any-url'+page,
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if(!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var rows = $('tr[id^=\'attachment-\']');
async.eachLimit(rows, 1, function(row, cb)
{
var id = row.attribs['id'];
var file = row.attribs['data-attachment-filename'];
var author = $(row).children('.creator').text().trim();
var created = $(row).children('.created-date').text().trim();
created = moment.tz(created, 'MMM D, YYYY', 'Europe/Madrid').format();
var urlFile = 'simple-file' + $(row).children('.filename-column').children('.filename').attr('href');
var extension = row.attribs['data-attachment-filename'].split('.');
extension = extension[extension.length-1];
if(created<limitDate && validExtensions.indexOf(extension)>-1)
{
var f = '{ "id": "' + id + '",';
f += ' "file": "' + file + '",';
f += ' "url": "' + urlFile + '",';
f += ' "author": "' + author + '",';
f += ' "modified": "' + created + '" }';
files.push(JSON.parse(f));
}
cb();
}, (err) => {
if(err) cbGetFiles(err);
});
cb();
} else {
cb(err);
}
});
}, function(err){
if(err){
cbGetFiles(err);
} else {
cbGetFiles(null, files);
}
});
},
function(files, cbGetAutors)
{
var filesFinal = {};
for(var f in files)
{
if(!filesFinal[files[f].author])
{
var ff = {};
for(var i in files)
{
if(files[i].author === files[f].author)
{
ff[files[i].file] = files[i].url;
}
}
filesFinal[files[f].author] = ff;
}
}
cbGetAutors(null, JSON.parse(JSON.stringify(filesFinal)));
},
function(filesFinal, cbSendEmail)
{
var authors = Object.keys(filesFinal);
async.eachSeries(authors, function(author, cb)
{
var name = author.split(' ');
var email = 'simple-mail#gmail.com';
var msg = '<p>Hi ' + author + ',</p><p>how is it going:</p><p>';
for(var a in Object.keys(filesFinal[author]))
{
msg += '<p style="margin-left:20px"> '+ICON_DOC+' <a href="';
msg += filesFinal[author][Object.keys(filesFinal[author])[a]]+'">'+Object.keys(filesFinal[author])[a]+'</a></p>';
}
msg += '</p></p><p><b>NOTE: This is a no-reply address.</b></p><p>Have a nice day! '+ICON_MONKEY+'</p>';
var message = {
text: msg,
from: 'test#mail.com',
to: email,
bcc: '',
subject: 'Sample subject',
attachment: [{data: msg, alternative: true}]
};
serverEmail.send(message, function(err, message)
{
if(err)
{
cb(err);
} else {
console.log(message);
cb();
}
});
}, function(err){
if(err) cbSendEmail(err);
cbSendEmail();
});
}
], (err) => {
if(err) console.log(err);
});
I would like to know if there is a way to control this issue or at least if there are another options for what I want to do.
Thanks.
A better (neat) way to use async waterfall.
Make sure you use return before any callback function. I have added them in the code.
Also, if you are nesting eachSeries, it is better to give a different name to callback function than the parent callback function.
I have changed 'cb' of child async.series to 'inner_cb'
Updated Code:
async.waterfall
([
funcOne,
funcTwo,
funcThree,
funcFour
], (err) => {
if(err) console.log(err);
});
funciton funcOne(cbNumPages) {
request({
url: 'any-url',
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if (!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var pagesList = $('ol.aui-nav').children();
if(pagesList.length<1)
{
var numPages = 1;
} else {
var numPages = pagesList.length-2;
}
console.log(numPages);
var pages = new Array(numPages),
total = numPages*20,
iterator = 0;
async.eachSeries(pages, function(page, cb)
{
if(page>1)
{
pages[iterator] = iterator;
}else {
pages[iterator] = iterator*20;
}
iterator++;
return cb();
}, function(err){
if(err) return cbNumPages(err);
return cbNumPages(null, pages);
});
} else {
return cbNumPages(err);
}
})
}
function funcTwo(pages, cbGetFiles) {
var files = [];
var limitDate = moment().tz('Europe/Madrid').subtract(330,'days').format();
async.eachSeries(pages, function(page, cb)
{
request({
url: 'any-url'+page,
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if(!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var rows = $('tr[id^=\'attachment-\']');
async.eachLimit(rows, 1, function(row, inner_cb)
{
var id = row.attribs['id'];
var file = row.attribs['data-attachment-filename'];
var author = $(row).children('.creator').text().trim();
var created = $(row).children('.created-date').text().trim();
created = moment.tz(created, 'MMM D, YYYY', 'Europe/Madrid').format();
var urlFile = 'simple-file' + $(row).children('.filename-column').children('.filename').attr('href');
var extension = row.attribs['data-attachment-filename'].split('.');
extension = extension[extension.length-1];
if(created<limitDate && validExtensions.indexOf(extension)>-1)
{
var f = '{ "id": "' + id + '",';
f += ' "file": "' + file + '",';
f += ' "url": "' + urlFile + '",';
f += ' "author": "' + author + '",';
f += ' "modified": "' + created + '" }';
files.push(JSON.parse(f));
}
return inner_cb();
}, (err) => {
if(err) return cbGetFiles(err);
});
return cb();
} else {
return cb(err);
}
});
}, function(err){
if(err){
return cbGetFiles(err);
} else {
return cbGetFiles(null, files);
}
});
}
function funcThree(files, cbGetAutors) {
var filesFinal = {};
for(var f in files)
{
if(!filesFinal[files[f].author])
{
var ff = {};
for(var i in files)
{
if(files[i].author === files[f].author)
{
ff[files[i].file] = files[i].url;
}
}
filesFinal[files[f].author] = ff;
}
}
return cbGetAutors(null, JSON.parse(JSON.stringify(filesFinal)));
}
function funcFour(filesFinal, cbSendEmail) {
var authors = Object.keys(filesFinal);
async.eachSeries(authors, function(author, cb)
{
var name = author.split(' ');
var email = 'simple-mail#gmail.com';
var msg = '<p>Hi ' + author + ',</p><p>how is it going:</p><p>';
for(var a in Object.keys(filesFinal[author]))
{
msg += '<p style="margin-left:20px"> '+ICON_DOC+' <a href="';
msg += filesFinal[author][Object.keys(filesFinal[author])[a]]+'">'+Object.keys(filesFinal[author])[a]+'</a></p>';
}
msg += '</p></p><p><b>NOTE: This is a no-reply address.</b></p><p>Have a nice day! '+ICON_MONKEY+'</p>';
var message = {
text: msg,
from: 'test#mail.com',
to: email,
bcc: '',
subject: 'Sample subject',
attachment: [{data: msg, alternative: true}]
};
serverEmail.send(message, function(err, message)
{
if(err)
{
return cb(err);
} else {
console.log(message);
return cb();
}
});
}, function(err){
if(err) return cbSendEmail(err);
return cbSendEmail();
});
}
As #YSK said in a comment, I was obtaining a 401 from the response.statusCode and therefore it is being missleaded to the cbSendEmail(err) with err beying null. Making the next method in the waterfall's first parameter beying the callback instead of the second.
I have a remote method in loopback like:
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if (alertRel.length > 0 && alertRel[0].alerts()) {
response.code = 200;
response.status = "success";
response.data = {};
if (alertRel[0].alertId) {
response.data.alertId = alertRel[0].alertId;
}
if (alertRel[0].readStatus) {
response.data.readStatus = alertRel[0].readStatus;
}
if (alertRel[0].receivedOn) {
response.data.alertReceivedOn = alertRel[0].receivedOn;
}
var alertData = alertRel[0].alerts();
if (alertData.title) {
response.data.alertTitle = alertData.title;
}
if (alertData.message) {
response.data.alertShortMessage = alertData.message;
}
if (alertData.extraMessage) {
response.data.alertMessage = alertData.extraMessage;
}
if (alertData.priority) {
response.data.alertPriority = alertData.priority;
}
if (alertData.validUntil) {
response.data.alertExpiresOn = alertData.validUntil;
}
if (alertData.images && alertData.images.length > 0) {
response.data.alertImages = [];
for (var image in alertData.images) {
if (alertData.images.hasOwnProperty(image)) {
response.data.alertImages.push(constants.ALERT_IMAGE_URL + '/' + alertData.images[image]);
}
}
}
if (alertData.alertTypes() && alertData.alertTypes().alertTypeName) {
response.data.alertType = alertData.alertTypes().alertTypeName;
}
if (alertData.alertLocations && alertData.alertLocations > 0) {
response.data.alertLocations = [];
response.data.policeDepartments = [];
response.data.hospitals = [];
response.data.fireDepartments = [];
var locations = alertData.alertLocations;
for (var locKey in locations) {
if (locations.hasOwnProperty(locKey)) {
if (locations[locKey].data) {
response.data.alertLocations.push(locations[locKey].data);
console.log(locations[locKey].data);
if (locations[locKey].data.type) {
var locationType = locations[locKey].data.type;
if (locationType === "Polygon") {
var coordinates = locations[locKey].data.coordinates[0];
var polygonCenter = getPolygonCenter(coordinates);
console.log(polygonCenter);
}
}
}
}
}
}
cb(null, response);
} else {
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
})
};
But when I call this method through api, response is received without data added from the complex code part. I know that callback will be called asynchronously here and so that cb(response) will be called before the complex code is executed completely. How can i send response only after the complex part is completed and data is correctly added to response from that data. I cannot move cb(response) inside the complex part as data is being pushed in for loop.
I have heard of promises, can it be used here, if so, how could it be done?
Someone please help!!
The problem is because of fetching relation in if.
The relation method is an async.
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if(alertRel.length < 1){
return handleError();
}
alertRel[0].alerts(handleResponse);
function handleResponse(err, alertRelAlert){
if(err) return handleError();
if (alertRelAlert) {
//all that code in question if if section
}else {
return handleError();
}
}
function handleError(){
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
});
}
I want to execute the callback in the else of my each loop. In my console i have the "Found" written but the callback isn't executed ...
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(jsonparse);
}
});
},
function(jsonparse){
console.log("In the 2nd loop !");
}
]);
can you try this
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(null,jsonparse);
}
});
},
function(jsonparse,callback){
console.log("In the 2nd loop !");
callback(null,'result');
}
],function(err,data){
//your data is result
//......Your function script
});
I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();