Loop callback in async.waterfall node js - node.js

I want to execute the callback in the else of my each loop. In my console i have the "Found" written but the callback isn't executed ...
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(jsonparse);
}
});
},
function(jsonparse){
console.log("In the 2nd loop !");
}
]);

can you try this
async.waterfall([
function readFile(callback){
console.log("Start async");
var params = {Bucket : "MyBucket", Key: "MyKey"};
reads3.getObject(params, function extract(err,data) {
//read a json object
console.log("Start reading");
callback(err,data);
});
},
function(data, callback){
var content = data.Body.toString('utf-8').trim();
var jsonparse = JSON.parse(content);
async.each(config, function(item) {
var currentPath = item.path;
if((key.search(currentPath)) === (-1)) {
console.log("No found !");
} else {
console.log("Found");
callback(null,jsonparse);
}
});
},
function(jsonparse,callback){
console.log("In the 2nd loop !");
callback(null,'result');
}
],function(err,data){
//your data is result
//......Your function script
});

Related

Parent function is not waiting for child to finish

Inside getMetadata(), Object.keys function is not waiting for httprequest to finish. How can I make object.keys function to wait till httprequest function manipluates the result variable?
I'm using node. I tried to make promise but failed.
function fetchAirportPageIDsListWithMetaJSON(faa, cb){
logger.info('[airportcms-data-processor] fetching airport pages list with Metadata');
var faa = faa
async.waterfall([
getAirportPageIDsList,
getMetadata,
], function (err, result) {
cb(null, result);
});
function getAirportPageIDsList(callback) {
httpRequests.fetchData('//s3 url to fetch data', function (err, data) {
var idsMap={};
data["page-ids"].forEach( (obj) => {
obj.list.forEach((item) => idsMap[item] = obj.id);
});
callback(null, idsMap);
})
}
function getMetadata(data,callback) {
var result=[];
Object.keys(data).sort().forEach( function (t) {
var tempJson={};
var urlcheck = verifySpecialPageId(t);
if (urlcheck){
var url = config.urls.s3_airports_base_url+'/'+faa+'/'+urlcheck;
}else{
var url = config.urls.s3_airports_base_url+'/'+faa+'/'+t;
}
tempJson["sectionId"]= t;
tempJson["page"]= data[t];
httpRequests.makeHeadRequestWithCallerId(url, function (err, metax) {
if (metax){
let z = metax.split('|')[0];
tempJson["SummaryRange"]= getSummaryRangeAirportPageList(z);
tempJson["timestamp"]= new Date(parseInt(z)).toLocaleDateString();
tempJson["callerId"]= metax.split('|')[1];
}else{
tempJson["timestamp"]='';
tempJson["callerId"]='';
tempJson["SummaryRange"]='';
}
})
result.push(tempJson);
});
logger.info("Final result: ", result);
callback(null, result);
}
}
http request function:
function makeHeadRequestWithCallerId (url, cb) {
httpRequest.head(url, function (err, res) {
if (err) {
logger.error('Error ' + err);
return cb(err, null);
}
if(res.code === 200) {
if (res.headers['x-amz-meta-x-amz-meta-lastmodified'] || res.headers['x-amz-meta-x-amz-meta-callerid']) {
var dateModified = res.headers['x-amz-meta-x-amz-meta-lastmodified'];
var timeStamp = Date.parse(dateModified);
var callerid = res.headers['x-amz-meta-x-amz-meta-callerid'];
if(timeStamp && callerid) {
return cb(null, timeStamp+'|'+callerid);
} else if (callerid){
return cb(null, '|'+callerid);
}else if(timeStamp){
return cb(null, timeStamp+'|');
}else{
return cb(null, null);
}
}else{
return cb(null, null);
}
}
});
}
Current log=> Final result:
[{ sectionId: 'internet-wifi', page: 'internet-wifi' },
{ sectionId: 'layover-ideas', page: 'layover-ideas' }]
Expected log=> Final result:
{ sectionId: 'internet-wifi',
page: 'internet-wifi',
SummaryRange: '12-99',
timestamp: '1/29/2018',
callerId: '' },
{ sectionId: 'layover-ideas',
page: 'layover-ideas',
SummaryRange: '12-99',
timestamp: '1/26/2017',
callerId: '' },
function getMetadata(data, callback) {
var result = [];
var count = Object.keys(data).length;
var i = 0;
Object.keys(data).sort().forEach(function (t) {
var tempJson = {};
var urlcheck = verifySpecialPageId(t);
if (urlcheck) {
var url = config.urls.s3_airports_base_url + '/' + faa + '/' + urlcheck;
} else {
var url = config.urls.s3_airports_base_url + '/' + faa + '/' + t;
}
tempJson["sectionId"] = t;
tempJson["page"] = data[t];
httpRequests.makeHeadRequestWithCallerId(url, function (err, metax) {
if (metax) {
let z = metax.split('|')[0];
tempJson["SummaryRange"] = getSummaryRangeAirportPageList(z);
tempJson["timestamp"] = new Date(parseInt(z)).toLocaleDateString();
tempJson["callerId"] = metax.split('|')[1];
} else {
tempJson["timestamp"] = '';
tempJson["callerId"] = '';
tempJson["SummaryRange"] = '';
}
result.push(tempJson);
i++;
if(count === i){
logger.info("Final result: ", result);
callback(null, result);
}
})
});
}

Nodejs/Async: How does callback work in iteratee function for async.map as mentioned in code snippet

Being new to nodejs ans async following is the code that I came across.
app = express();
/*
other express use calls like - app.use(bodyParser.json());
*/
var async = require("async");
var server;
app.post('/callType/call', function(req, res) {
var startTime = Date.now();
server = req.body.server;
//async.map asynchronuously call enrollStep1 for every element in the req.body.nodes array
//HOW DOES THIS WORK??!! - WHERE IS THE CALLBACK DEFINED OR SOURCED FROM???
//******************************************************
async.map(req.body.nodes, function(node, callback) {
someFunc(node.property1,node.property2,callback)
},
//This function is called when every task triggered by async.map has called its callback.
function(err, results) {
var response = {};
if (err) {
response.success = false;
response.error = err;
console.log("ERROR returned: " + JSON.stringify(response));
res.json(response);
} else {
var returnResults = [];
//Results is an array of array - flatten it
var flattenedResults = [].concat.apply([], results);
//then remove duplicates
for (var i = 0; i < flattenedResults.length; i++){
var obj = flattenedResults[i];
var isDup = returnResults.some(function(element) {
return element.tid === obj.tid;
});
if (!isDup) {
returnResults.push(obj);
}
}
response.success = true;
response.results = returnResults;
res.json(response);
}
});
});
function someFunc(property1, property2, callback) {
var url = '/'+callTypes +'/'+ call +'/'+ property1 +'/'+ property2
urClient
.get(server + url)
.header('Content-Type', 'application/json')
.end(
function(response) {
if (response.code !== 200) {
callback("Error " + ". Code: " + response.code + " Response: " + JSON.stringify(response));
} else {
callback("Success " + ". Code: " + response.code + " Response: " + JSON.stringify(response));
}
}
);
}
The iteratee function for async.map has a definition starting function(node, callback) { but the callback function is never assigned. How does the callback work over here.
Isn't it supposed to be assigned somewhere like callback = myCallbackFunction;
The async.map takes 3 arguments, the array/object, the function to map the data and the callback function, so your code should be:
async.map(req.body.nodes, someFunc , function(err, results) {
if (err) return console.log(err);
console.log(results);
});
And your someFunc should be:
function someFunc(item, callback) {
// do something with item
// it's each item in the original array/object
callback('The results');
}
This is a basic example: http://code.runnable.com/UyR-6c2DZZ4SmfSh/async-map-example-for-node-js

Completing loops inside of node.js promises

I am trying to complete a few loops over firebase objects using .forEach and I am also using promises. This isn't working out how I had planned it. My basic problem is that the loops inside of my promises complete well after the promise chain itself completes. Here is my function:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "";
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
console.log('one');
progress(20);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
console.log('two');
progress(40);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
console.log('three');
progress(60);
});
}).then(function() {
posts.once('value', function(postSnapshot) {
// use Promise.all and Array#map to wait for all these queries to finish
var allPosts = postSnapshot.val()
var postKeys = Object.keys(allPosts)
return Promise.all(postKeys.map(function(postKey) {
var postID = postKey;
return posts.child(postID).child('items').child(oldProductID).once('value', function(itemSnapshot) {
return itemSnapshot.forEach(function(itemSnapshot) {
var itemData = itemSnapshot.val()
console.log('post snapshot'+ itemData);
updateObject['posts/'+postID+'/items/'+oldProductID] = null
updateObject['posts/'+postID+'/items/'+newProductID] = itemData
});
});
}));
});
}).then(function() {
// Move to next item
return console.log('hey look here'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
}).then(function() {
// Move to next item
return console.log('second one'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});
And here is the output:
one
two
three
hey look hereundefined
second oneundefined
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
I think I might be using promises incorrectly but I don't know.
I figured out that I should be using Promise.all() in order to wait for my forEach loops to complete. Here is the code I used to solve my problem, enjoy:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "-KZOO0UII67uOmYo6DJh";
var postKeys = [];
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
return clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
progress(10);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
progress(25);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
progress(40);
});
}).then(function() {
progress(55);
return posts.orderByChild('receipt').equalTo(receiptID).once('value');
}).then(function(postSnapshot) {
return postSnapshot.forEach(function(post) {
progress(70);
postKeys.push(post.key)
});
}).then(function() {
return Promise.all(postKeys.map(function(postKey) {
return posts.child(postKey).child('items').child(oldProductID).once('value', function(itemSnapshot) {
var itemData = itemSnapshot.val()
updateObject['posts/'+postKey+'/items/'+oldProductID] = null;
updateObject['posts/'+postKey+'/items/'+newProductID] = itemData;
});
})).then(function(results) {
progress(85);
return results;
});
}).then(function() {
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
resolve();
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});

Waiting for asynchronous task to complete nodejs

I am creating an application where a user can have many rooms and each room can have many channels, here is my code when retrieving the rooms and corresponding channels:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
async.each(rooms, function (roomId, cb){
roomsObj = {};
if (roomId !== '') {
Rooms.findOne({id: roomId}).exec(function (err, room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
var channels = room.channels;
async.each(channels, function (channelId, cb) {
chansObj = {};
Channels.findOne({id: channelId}).exec(function (err, channel){
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
cb();
});
},
function (err) {
});
});
}
cb();
}, function (err) {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log(roomsArr);
});
});
}
It is suppose to return a javascript object with the following structure:
[ { name: "Room Name",
roomId: "Room Id",
inviteLink: "Room Invite Link",
channels: [
{
name: "Channel Name",
id: "channel Id"
}
]
}
]
But I always get an empty array because async.each(rooms, function (roomId, cb){ }) does not wait for async.each(channels, function (channelId, cb) {}) to complete, so I have empty room object. Please how do I solve this issue ?
You should call your rooms's callback loop after completing you channels loop.
You should do something like this:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
async.each(rooms, function (roomId, callback1){
roomsObj = {};
if (roomId !== '') {
Rooms.findOne({id: roomId}).exec(function (err, room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
var channels = room.channels;
var i=0;
async.each(channels, function (channelId, callback2) {
chansObj = {};
Channels.findOne({id: channelId}).exec(function (err, channel){
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
i++;
if(i===(channels.length-1)){
i=0;
callback1();
}else{
callback2();
}
});
},
function (err) {
});
});
}
}, function (err) {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log(roomsArr);
});
});
}
I solved it, it really was a case for promises, I used bluebird promise combined with async - the modified code:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
Promise.each(rooms, function (roomId, callback1){
roomsObj = {};
if (roomId !== '') {
async.series ([
function () {
Rooms.findOne({id: roomId}).then(function (room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
channels = room.channels;
sails.log(roomName);
})
}
]);
return Promise.each(channels, function (channelId) {
return Promise.all([
Channels.findOne({id: channelId}).then(function (channel){
chansObj = {};
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
sails.log(chansObj);
})
]).then(function () {
sails.log('done one');
});
}).then(function () {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log('done all');
chansArr = [];
});
}
}).then(function () {
sails.log(roomsArr);
sails.log("grand finish");
});
});
}
Thanks to everyone who contributed.

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Resources