Parent function is not waiting for child to finish - node.js

Inside getMetadata(), Object.keys function is not waiting for httprequest to finish. How can I make object.keys function to wait till httprequest function manipluates the result variable?
I'm using node. I tried to make promise but failed.
function fetchAirportPageIDsListWithMetaJSON(faa, cb){
logger.info('[airportcms-data-processor] fetching airport pages list with Metadata');
var faa = faa
async.waterfall([
getAirportPageIDsList,
getMetadata,
], function (err, result) {
cb(null, result);
});
function getAirportPageIDsList(callback) {
httpRequests.fetchData('//s3 url to fetch data', function (err, data) {
var idsMap={};
data["page-ids"].forEach( (obj) => {
obj.list.forEach((item) => idsMap[item] = obj.id);
});
callback(null, idsMap);
})
}
function getMetadata(data,callback) {
var result=[];
Object.keys(data).sort().forEach( function (t) {
var tempJson={};
var urlcheck = verifySpecialPageId(t);
if (urlcheck){
var url = config.urls.s3_airports_base_url+'/'+faa+'/'+urlcheck;
}else{
var url = config.urls.s3_airports_base_url+'/'+faa+'/'+t;
}
tempJson["sectionId"]= t;
tempJson["page"]= data[t];
httpRequests.makeHeadRequestWithCallerId(url, function (err, metax) {
if (metax){
let z = metax.split('|')[0];
tempJson["SummaryRange"]= getSummaryRangeAirportPageList(z);
tempJson["timestamp"]= new Date(parseInt(z)).toLocaleDateString();
tempJson["callerId"]= metax.split('|')[1];
}else{
tempJson["timestamp"]='';
tempJson["callerId"]='';
tempJson["SummaryRange"]='';
}
})
result.push(tempJson);
});
logger.info("Final result: ", result);
callback(null, result);
}
}
http request function:
function makeHeadRequestWithCallerId (url, cb) {
httpRequest.head(url, function (err, res) {
if (err) {
logger.error('Error ' + err);
return cb(err, null);
}
if(res.code === 200) {
if (res.headers['x-amz-meta-x-amz-meta-lastmodified'] || res.headers['x-amz-meta-x-amz-meta-callerid']) {
var dateModified = res.headers['x-amz-meta-x-amz-meta-lastmodified'];
var timeStamp = Date.parse(dateModified);
var callerid = res.headers['x-amz-meta-x-amz-meta-callerid'];
if(timeStamp && callerid) {
return cb(null, timeStamp+'|'+callerid);
} else if (callerid){
return cb(null, '|'+callerid);
}else if(timeStamp){
return cb(null, timeStamp+'|');
}else{
return cb(null, null);
}
}else{
return cb(null, null);
}
}
});
}
Current log=> Final result:
[{ sectionId: 'internet-wifi', page: 'internet-wifi' },
{ sectionId: 'layover-ideas', page: 'layover-ideas' }]
Expected log=> Final result:
{ sectionId: 'internet-wifi',
page: 'internet-wifi',
SummaryRange: '12-99',
timestamp: '1/29/2018',
callerId: '' },
{ sectionId: 'layover-ideas',
page: 'layover-ideas',
SummaryRange: '12-99',
timestamp: '1/26/2017',
callerId: '' },

function getMetadata(data, callback) {
var result = [];
var count = Object.keys(data).length;
var i = 0;
Object.keys(data).sort().forEach(function (t) {
var tempJson = {};
var urlcheck = verifySpecialPageId(t);
if (urlcheck) {
var url = config.urls.s3_airports_base_url + '/' + faa + '/' + urlcheck;
} else {
var url = config.urls.s3_airports_base_url + '/' + faa + '/' + t;
}
tempJson["sectionId"] = t;
tempJson["page"] = data[t];
httpRequests.makeHeadRequestWithCallerId(url, function (err, metax) {
if (metax) {
let z = metax.split('|')[0];
tempJson["SummaryRange"] = getSummaryRangeAirportPageList(z);
tempJson["timestamp"] = new Date(parseInt(z)).toLocaleDateString();
tempJson["callerId"] = metax.split('|')[1];
} else {
tempJson["timestamp"] = '';
tempJson["callerId"] = '';
tempJson["SummaryRange"] = '';
}
result.push(tempJson);
i++;
if(count === i){
logger.info("Final result: ", result);
callback(null, result);
}
})
});
}

Related

Node js Promises with recursive function

I want to read the all (text) files from a specific directory and it's all subdirecoty recursively.. I am able to read the file and append the result to a global variable. but i want to access the variable at the end of all operation. I am trying with promises but i am unable to access it. please help
var file_path = `C:\\Users\\HP\\Desktop\\test_folder`;
const fs = require('fs');
var final_array = [];
let getFolderTree = function(file_path) {
return new Promise(function(resolve, reject) {
fs.readdir(file_path, function(err, folders) {
if (err) {
console.log("error reading folder :: " + err);
} else {
if (folders.length !== 0) {
for (let i = 0; i < folders.length; i++) {
if (folders[i].endsWith("txt")) {
let text_file_path = file_path + `\\` + folders[i];
fs.readFile(text_file_path, function(error_read, data) {
if (error_read) {
console.log("error reading " + error_read);
} else {
return resolve(final_array.push(data));// want to access final_array at the end of all operations
}
});
} else {
let current_path = file_path + `\\` + folders[i];
getFolderTree(current_path);
}
}
}
}
});
});
}
getFolderTree(file_path).then(function() {
console.log(final_array); // this is not working
});
I think i have found the solution but I am still confused about how it works.
I took reference from another code and able to figure out some how.
var fs = require('fs');
var path = require('path');
let root_path = "C:\\Users\\HP\\Desktop\\test_folder";
function getAllDirectoriesPath(current_path) {
var results = [];
return new Promise(function (resolve, reject) {
fs.readdir(current_path, function (erro, sub_dirs) {
if (erro) {
console.log(error);
} else {
let no_of_subdir = sub_dirs.length;
if (!no_of_subdir) {
return resolve(results);
} else {
sub_dirs.forEach(function (dir) {
dir = path.resolve(current_path, dir);
fs.stat(dir, function (err, stat) {
if (stat && stat.isDirectory()) {
getAllDirectoriesPath(dir).then(function (res) {
results = results.concat(res);
if (!--no_of_subdir) {
resolve(results);
}
});
} else {
fs.readFile(dir, function (err, data) {
results.push(data.toString());
if (!--no_of_subdir) {
resolve(results);
}
});
}
});
});
}
}
});
});
}
getAllDirectoriesPath(root_path).then(function (results) {
console.log(results);
});

NodeJs async waterfall (callback method is not a function)

I am having a problem using async waterfall where I find that after calling the second callback (cbNumPages), the first parameter "pages" is the actual callback for the next function, instead of the last parameter "cbGetFiles" which it should be (as far as I know async waterfall says that last parameter should always be the callback, well in this case is apparently not).
The code is the following:
async.waterfall
([
function(cbNumPages)
{
request({
url: 'any-url',
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if (!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var pagesList = $('ol.aui-nav').children();
if(pagesList.length<1)
{
var numPages = 1;
} else {
var numPages = pagesList.length-2;
}
console.log(numPages);
var pages = new Array(numPages),
total = numPages*20,
iterator = 0;
async.eachSeries(pages, function(page, cb)
{
if(page>1)
{
pages[iterator] = iterator;
}else {
pages[iterator] = iterator*20;
}
iterator++;
cb();
}, function(err){
if(err) cbNumPages(err);
cbNumPages(null, pages);
});
} else {
cbNumPages(err);
}
})
},
function(pages, cbGetFiles)
{
var files = [];
var limitDate = moment().tz('Europe/Madrid').subtract(330,'days').format();
async.eachSeries(pages, function(page, cb)
{
request({
url: 'any-url'+page,
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if(!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var rows = $('tr[id^=\'attachment-\']');
async.eachLimit(rows, 1, function(row, cb)
{
var id = row.attribs['id'];
var file = row.attribs['data-attachment-filename'];
var author = $(row).children('.creator').text().trim();
var created = $(row).children('.created-date').text().trim();
created = moment.tz(created, 'MMM D, YYYY', 'Europe/Madrid').format();
var urlFile = 'simple-file' + $(row).children('.filename-column').children('.filename').attr('href');
var extension = row.attribs['data-attachment-filename'].split('.');
extension = extension[extension.length-1];
if(created<limitDate && validExtensions.indexOf(extension)>-1)
{
var f = '{ "id": "' + id + '",';
f += ' "file": "' + file + '",';
f += ' "url": "' + urlFile + '",';
f += ' "author": "' + author + '",';
f += ' "modified": "' + created + '" }';
files.push(JSON.parse(f));
}
cb();
}, (err) => {
if(err) cbGetFiles(err);
});
cb();
} else {
cb(err);
}
});
}, function(err){
if(err){
cbGetFiles(err);
} else {
cbGetFiles(null, files);
}
});
},
function(files, cbGetAutors)
{
var filesFinal = {};
for(var f in files)
{
if(!filesFinal[files[f].author])
{
var ff = {};
for(var i in files)
{
if(files[i].author === files[f].author)
{
ff[files[i].file] = files[i].url;
}
}
filesFinal[files[f].author] = ff;
}
}
cbGetAutors(null, JSON.parse(JSON.stringify(filesFinal)));
},
function(filesFinal, cbSendEmail)
{
var authors = Object.keys(filesFinal);
async.eachSeries(authors, function(author, cb)
{
var name = author.split(' ');
var email = 'simple-mail#gmail.com';
var msg = '<p>Hi ' + author + ',</p><p>how is it going:</p><p>';
for(var a in Object.keys(filesFinal[author]))
{
msg += '<p style="margin-left:20px"> '+ICON_DOC+' <a href="';
msg += filesFinal[author][Object.keys(filesFinal[author])[a]]+'">'+Object.keys(filesFinal[author])[a]+'</a></p>';
}
msg += '</p></p><p><b>NOTE: This is a no-reply address.</b></p><p>Have a nice day! '+ICON_MONKEY+'</p>';
var message = {
text: msg,
from: 'test#mail.com',
to: email,
bcc: '',
subject: 'Sample subject',
attachment: [{data: msg, alternative: true}]
};
serverEmail.send(message, function(err, message)
{
if(err)
{
cb(err);
} else {
console.log(message);
cb();
}
});
}, function(err){
if(err) cbSendEmail(err);
cbSendEmail();
});
}
], (err) => {
if(err) console.log(err);
});
I would like to know if there is a way to control this issue or at least if there are another options for what I want to do.
Thanks.
A better (neat) way to use async waterfall.
Make sure you use return before any callback function. I have added them in the code.
Also, if you are nesting eachSeries, it is better to give a different name to callback function than the parent callback function.
I have changed 'cb' of child async.series to 'inner_cb'
Updated Code:
async.waterfall
([
funcOne,
funcTwo,
funcThree,
funcFour
], (err) => {
if(err) console.log(err);
});
funciton funcOne(cbNumPages) {
request({
url: 'any-url',
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if (!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var pagesList = $('ol.aui-nav').children();
if(pagesList.length<1)
{
var numPages = 1;
} else {
var numPages = pagesList.length-2;
}
console.log(numPages);
var pages = new Array(numPages),
total = numPages*20,
iterator = 0;
async.eachSeries(pages, function(page, cb)
{
if(page>1)
{
pages[iterator] = iterator;
}else {
pages[iterator] = iterator*20;
}
iterator++;
return cb();
}, function(err){
if(err) return cbNumPages(err);
return cbNumPages(null, pages);
});
} else {
return cbNumPages(err);
}
})
}
function funcTwo(pages, cbGetFiles) {
var files = [];
var limitDate = moment().tz('Europe/Madrid').subtract(330,'days').format();
async.eachSeries(pages, function(page, cb)
{
request({
url: 'any-url'+page,
qs: {},
method: 'GET',
headers: {
'Authorization' : 'any-auth'
}
}, (err, response, body) => {
if(!err && response.statusCode == 200)
{
var $ = cheerio.load(body);
var rows = $('tr[id^=\'attachment-\']');
async.eachLimit(rows, 1, function(row, inner_cb)
{
var id = row.attribs['id'];
var file = row.attribs['data-attachment-filename'];
var author = $(row).children('.creator').text().trim();
var created = $(row).children('.created-date').text().trim();
created = moment.tz(created, 'MMM D, YYYY', 'Europe/Madrid').format();
var urlFile = 'simple-file' + $(row).children('.filename-column').children('.filename').attr('href');
var extension = row.attribs['data-attachment-filename'].split('.');
extension = extension[extension.length-1];
if(created<limitDate && validExtensions.indexOf(extension)>-1)
{
var f = '{ "id": "' + id + '",';
f += ' "file": "' + file + '",';
f += ' "url": "' + urlFile + '",';
f += ' "author": "' + author + '",';
f += ' "modified": "' + created + '" }';
files.push(JSON.parse(f));
}
return inner_cb();
}, (err) => {
if(err) return cbGetFiles(err);
});
return cb();
} else {
return cb(err);
}
});
}, function(err){
if(err){
return cbGetFiles(err);
} else {
return cbGetFiles(null, files);
}
});
}
function funcThree(files, cbGetAutors) {
var filesFinal = {};
for(var f in files)
{
if(!filesFinal[files[f].author])
{
var ff = {};
for(var i in files)
{
if(files[i].author === files[f].author)
{
ff[files[i].file] = files[i].url;
}
}
filesFinal[files[f].author] = ff;
}
}
return cbGetAutors(null, JSON.parse(JSON.stringify(filesFinal)));
}
function funcFour(filesFinal, cbSendEmail) {
var authors = Object.keys(filesFinal);
async.eachSeries(authors, function(author, cb)
{
var name = author.split(' ');
var email = 'simple-mail#gmail.com';
var msg = '<p>Hi ' + author + ',</p><p>how is it going:</p><p>';
for(var a in Object.keys(filesFinal[author]))
{
msg += '<p style="margin-left:20px"> '+ICON_DOC+' <a href="';
msg += filesFinal[author][Object.keys(filesFinal[author])[a]]+'">'+Object.keys(filesFinal[author])[a]+'</a></p>';
}
msg += '</p></p><p><b>NOTE: This is a no-reply address.</b></p><p>Have a nice day! '+ICON_MONKEY+'</p>';
var message = {
text: msg,
from: 'test#mail.com',
to: email,
bcc: '',
subject: 'Sample subject',
attachment: [{data: msg, alternative: true}]
};
serverEmail.send(message, function(err, message)
{
if(err)
{
return cb(err);
} else {
console.log(message);
return cb();
}
});
}, function(err){
if(err) return cbSendEmail(err);
return cbSendEmail();
});
}
As #YSK said in a comment, I was obtaining a 401 from the response.statusCode and therefore it is being missleaded to the cbSendEmail(err) with err beying null. Making the next method in the waterfall's first parameter beying the callback instead of the second.

How to send a response only after a query has been executed in loopback

I have a remote method in loopback like:
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if (alertRel.length > 0 && alertRel[0].alerts()) {
response.code = 200;
response.status = "success";
response.data = {};
if (alertRel[0].alertId) {
response.data.alertId = alertRel[0].alertId;
}
if (alertRel[0].readStatus) {
response.data.readStatus = alertRel[0].readStatus;
}
if (alertRel[0].receivedOn) {
response.data.alertReceivedOn = alertRel[0].receivedOn;
}
var alertData = alertRel[0].alerts();
if (alertData.title) {
response.data.alertTitle = alertData.title;
}
if (alertData.message) {
response.data.alertShortMessage = alertData.message;
}
if (alertData.extraMessage) {
response.data.alertMessage = alertData.extraMessage;
}
if (alertData.priority) {
response.data.alertPriority = alertData.priority;
}
if (alertData.validUntil) {
response.data.alertExpiresOn = alertData.validUntil;
}
if (alertData.images && alertData.images.length > 0) {
response.data.alertImages = [];
for (var image in alertData.images) {
if (alertData.images.hasOwnProperty(image)) {
response.data.alertImages.push(constants.ALERT_IMAGE_URL + '/' + alertData.images[image]);
}
}
}
if (alertData.alertTypes() && alertData.alertTypes().alertTypeName) {
response.data.alertType = alertData.alertTypes().alertTypeName;
}
if (alertData.alertLocations && alertData.alertLocations > 0) {
response.data.alertLocations = [];
response.data.policeDepartments = [];
response.data.hospitals = [];
response.data.fireDepartments = [];
var locations = alertData.alertLocations;
for (var locKey in locations) {
if (locations.hasOwnProperty(locKey)) {
if (locations[locKey].data) {
response.data.alertLocations.push(locations[locKey].data);
console.log(locations[locKey].data);
if (locations[locKey].data.type) {
var locationType = locations[locKey].data.type;
if (locationType === "Polygon") {
var coordinates = locations[locKey].data.coordinates[0];
var polygonCenter = getPolygonCenter(coordinates);
console.log(polygonCenter);
}
}
}
}
}
}
cb(null, response);
} else {
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
})
};
But when I call this method through api, response is received without data added from the complex code part. I know that callback will be called asynchronously here and so that cb(response) will be called before the complex code is executed completely. How can i send response only after the complex part is completed and data is correctly added to response from that data. I cannot move cb(response) inside the complex part as data is being pushed in for loop.
I have heard of promises, can it be used here, if so, how could it be done?
Someone please help!!
The problem is because of fetching relation in if.
The relation method is an async.
Alerts.getAlertDetails = function (alertId, options, cb) {
var response = {};
var userId = options.accessToken.userId;
Alerts.app.models.MobileUserAlertRelation.find({where: {userId: userId, alertId: alertId, isDeleted: -1}, include: {relation: 'alerts', scope: {include: ['alertTypes'], where: {status: 1}}}}, function (err, alertRel) {
if(alertRel.length < 1){
return handleError();
}
alertRel[0].alerts(handleResponse);
function handleResponse(err, alertRelAlert){
if(err) return handleError();
if (alertRelAlert) {
//all that code in question if if section
}else {
return handleError();
}
}
function handleError(){
response.code = 404;
response.status = 'error';
response.message = 'Alert not found.';
cb(null, response);
}
});
}

Loop callback in async.waterfall node js

I want to execute the callback in the else of my each loop. In my console i have the "Found" written but the callback isn't executed ...
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(jsonparse);
}
});
},
function(jsonparse){
console.log("In the 2nd loop !");
}
]);
can you try this
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(null,jsonparse);
}
});
},
function(jsonparse,callback){
console.log("In the 2nd loop !");
callback(null,'result');
}
],function(err,data){
//your data is result
//......Your function script
});

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Resources