I'm new to Node.js and Async coding. I need to write the equivalent of a nested for loop which will work with Node. I understand that my question is very similar to the one posted here: nested loops asynchronusly in nodejs, next loop must start only after one gets completed, but even after looking at that post in detail, I was unable to fix my code.
I am working with an XML feed. The 'parser' uses the xml2js package. The
loop runs exactly as expected if I remove the sql query (for which I'm using the mysql node package), but when I put the sql query in, then all the orders get processed first, the the "DONE" is output, and then the query fails as it tries to look up items for just the last order repeatedly.
I've tried replacing the for loops with async.forEach loops, but this did not help.
Any help or advice on how to recode this in a way more idiomatic to node would be greatly appreciated.
Many thanks!
Sixhobbits
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var curr, currItem, currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
for (var j=0; j<numOrders; j++){
//current order
curr = result['Root']['Orders'][0]['Order'][j];
currOrdId = curr['OrderId'][0]
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
// for each item
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
});
}//for
}//for
console.log("DONE");
});//parseString
You were on the right track by looking to use async.forEach. Here's how you would rework this code to use that:
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
async.forEach(result['Root']['Orders'][0]['Order'], function (curr, callback1) {
currOrdId = curr['OrderId'][0];
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
async.forEach(items, function (currItem, callback2) {
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
callback2(err);
});
}, callback1);
}, function (err) {
console.log("DONE");
});
});//parseString
Each iteration of async.forEach must call its callback parameter when all of its async processing has completed. You've got two levels in this case which makes it a little more difficult to keep track of in your head, but it's the same concept.
This is a classic closure-in-a-loop problem. You need to break the closure by passing currItem as an argument:
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,(function(CI){
return function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",CI['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
}
})(currItem)); // Break closure by passing currItem as argument
}//for
I realize this is an old post, but you might find this function useful
eachKVAsync = function(elements,userInfo,onKeyValue,ondone) {
var onDone = ondone;
var ret = null;
var done=false;
var isArray = typeof elements.forEach===$f$;
var keys = isArray ? null : [],
values = isArray ? elements : [];
if (keys) {
for (var k in elements) {
keys.push(k);
values.push(elements[k]);
}
}
var aborted=false;
var endLoop = function (userInfo){
aborted=true;
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
}
}
var i = 0;
var iterate = function (userInfo) {
if (i < values.length) {
var ix=i;
i++;
onKeyValue((keys?keys[ix]:i),values[ix],userInfo,iterate,endLoop);
} else {
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
return;
}
}
}
iterate(userInfo);
},
use example
eachKVAsync(
elements, {
aValue: 2004
},
function onItem(key, value, info, nextItem, endLoop) {
if (value.isOk) {
info.aValue += value.total;
setTimeout(nextItem,1000,info);
} else {
endLoop(info);
}
},
function afterLastItem(info, loopEndedEarly) {
if (!loopEndedEarly) {
console.log(info.aValue);
}
}
);
Related
I have following two collection:
1) Users: {name: xyz,
email:abc#xuz.com}
2) Posts: {_id: 12345678,
time:asdfg,
authEmail:abc#xyz.com,
description: asdigligvilud}
Here i want to get the details of each post along with the corresponding name of author id.
So i wrote query like this:
// get all the posts
Posts.find((err, posts)=> {
if (err) { next(err) };
var data = Array();
var count = 0;
var len = posts.length;
// function to check the end of inner queries
var checkloop = function(){
count++;
if(count===len)
return res.json({result:data,
msg:'success'});
}
for(var i=0;i<len;i++){
Users.findOne({email:posts[count].authEmail},(err,usr)=>{
if(usr){
var item = {
'authorName':usr.name?
'email':usr.email,
'postDesc':posts[count].desc,
'creationTime':posts[count].time
}
data.push(item);
}
checkloop();
});
}
});
But i am getting same result for each iterations that is the post deatils and author id corresponding to the very first loop that is posts[0] only.
So i think its because of asynchronous nature of queries. Is there any standard way to make such query in mongodb or do i need to change my callback method?
Thanks in advance.
I believe the issue you had was because count inside the for loop probably was meant to be i. In addition, each loop iteration captures the same variable i in the Users.findOne callback, but the callback should be able to refer to the post that was queried. Without making much changes, I think this can work with using forEach over the returned posts like below:
// get all the posts
Posts.find((err, posts)=> {
if (err) { next(err) };
var data = Array();
var count = 0;
var len = posts.length;
// function to check the end of inner queries
var checkloop = function(){
count++;
if(count===len)
return res.json({result:data,
msg:'success'});
}
posts.forEach((post) => {
Users.findOne({email:post.authEmail},(err,usr)=>{
if(usr){
var item = {
'authorName':usr.name?
'email':usr.email,
'postDesc':post.desc,
'creationTime':post.time
}
data.push(item);
}
checkloop();
}
});
});
I'm making a bot for Discord, and one of the features in this bot is a level system. I've decided to go from using JSON to store the data, to sqlite. I'm using sqlite3 in node.js and am trying to create a function to create / retrieve a player's data. My goal is to make this function return the data from the query, but I'm running my head into a brick wall trying to figure out what I'm doing wrong. I've read that I need to use callbacks sent to the query function, but that has not worked for me either (nor will it work for the goal of this function).
So any help on how I can create a function to return the data from the query would be extremely helpful! This is the code I have so far before I've given up and decided to come here for help before I lose any more hair. Everything else is working as I intend, I am just unable to retrieve the data from the db.each().
exports.getPlayerData = function(players,msg,mention = false){
if(mention){
// console.log("mention found");
var member = msg.mentions.users.first();
} else {
var member = msg.member.user;
};
var db = new sqlite3.Database('data/levels/' + msg.guild.id + '.db');
var check;
db.serialize(function() {
var stmt = "CREATE TABLE if not exists uid_" + member.id + " (id INTEGER PRIMARY KEY, username TEXT, avatarID TEXT, avatarURL TEXT, xp INTEGER, level INTEGER, lastXp INTEGER)";
db.run(stmt);
db.run("INSERT OR IGNORE INTO uid_" + member.id + " (id,username,avatarURL,xp,level,lastXP) VALUES (?,?,?,?,?,?)", member.id,member.username, member.avatarURL , 0, 0, 0);
db.each("SELECT * FROM uid_" + member.id + " WHERE id = " + member.id , function(err, row) {
console.log(row);
return row;
});
});
// return row;
db.close();
}
If I understood your problem correctly, then you must be facing issue related to the placement of return statement. You want to return all rows, but aren't able to return, because of asynchronous nature of code.
I checked the documentation of sqlite3's .each, and found out that there exist complete callback that you can call after you have fetched all rows. In that callback, you can return data array that has collection of rows.
Update: You have to use callback
exports.getPlayerData = function(players,msg,mention = false, callback){
if(mention){
// console.log("mention found");
var member = msg.mentions.users.first();
} else {
var member = msg.member.user;
};
var db = new sqlite3.Database('data/levels/' + msg.guild.id + '.db');
var data = []; //for storing the rows.
db.serialize(function() {
var stmt = "CREATE TABLE if not exists uid_" + member.id + " (id INTEGER PRIMARY KEY, username TEXT, avatarID TEXT, avatarURL TEXT, xp INTEGER, level INTEGER, lastXp INTEGER)";
db.run(stmt);
db.run("INSERT OR IGNORE INTO uid_" + member.id + " (id,username,avatarURL,xp,level,lastXP) VALUES (?,?,?,?,?,?)", member.id,member.username, member.avatarURL , 0, 0, 0);
db.each("SELECT * FROM uid_" + member.id + " WHERE id = " + member.id , function(err, row) {
data.push(row); //pushing rows into array
}, function(){ // calling function when all rows have been pulled
db.close(); //closing connection
callback(data);
});
});
}
To have to pass callback to get player data.
getPlayerData(players, msg, false, function(data){
console.log(data);
});
Hope it works for you and saved your hair :). Let me know, if there are any issues.
You should go with wait.for npm.
More example can be found here: https://www.npmjs.com/package/wait.for
Sharing code which worked for me below you can find it :
var wait = req.app.get('wait');
wait.launchFiber(handleGet, req, res);
function handleGet(req, res)
{
var slashes = req.app.get('slashes');
var connection = req.app.get('connection');
var user_lib = req.app.get('user_lib');
var user_id = req.params.userId;
user_id = slashes.add(user_id);
var user_insert_id = wait.for(user_lib.update_user_status, connection, user_id);
res.render('view/email_verification', { req:req, res:res});
}
user code lib file :
Object.prototype.update_user_status=function (connection, cc_user_id, callback)
{
connection.query('update cc_user set cc_user_active="yes" where cc_user_id ="'+cc_user_id+'"',function(err,rows){
if(err) throw err;
if(rows)
{
return callback(err,{rows:rows});
}
});
}
I wrote a code using async.each for inserting data into tables.
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
async.each(categoryList, function(item,callback)
{
var categoryName=item.categoryName;
var productName=item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%"+categoryName+"%'");
model.client.query(categoryCheckQuery,function (err,result) {
if(result.rowCount==0){
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)",categoryName);
model.client.query(insertCategoryQuery,function (err,result) {
if (!err) {
console.log("success");
}
});
}
else{
//insert product into product table
}
});
Explanation:
1)Here first Json array containing categoryName->biriyani is entered into shop table
2)when async.each fetching next json array containing categoryName->biriyani,
categoryCheckQuery checks the shop table whether categoryname = 'biriyani' is already exists.
3)If exists it wont be saved
Problem:
Here for both the data result.rowCount ==0 and both the data which have categoryname = biriyani is entered into shop table.
There are couple problems in this code.
One is the use of async.each(), async.eachSeries() should be used instead of async.each() because the operation of the next item in categoryList depends on current item's operation. async.eachSeries() ensures that first item is done before moving on to next item.
Another is async.each()'s callback() should be called to signal it that you're done with it.
Here is the revised code:
var categoryList = [{"categoryName": "biriyani","productName":"chicken biriyani"}, {"categoryName":"biriyani","productName":"mutton biriyani"}]
// Use async.eachSerices() instead of async.each()
async.eachSeries(categoryList, function(item,callback) {
var categoryName = item.categoryName;
var productName = item.productName;
var categoryCheckQuery = pgFormat("select * from shop where categoryName LIKE '%" + categoryName + "%'");
model.client.query(categoryCheckQuery, function (err, result) {
if (result.rowCount == 0) {
var insertCategoryQuery = pgFormat("insert into shop(categoryName)values(%L)", categoryName);
model.client.query(insertCategoryQuery, function (err, result) {
if (!err) {
console.log("success");
}
// passing non-null if you want to stop async.eachSeries() in case of error
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
else {
//insert product into product table
doInsert(params, function(err, result) {
callback(null); // <<<<< need to call async.each()'s callback() here
});
}
});
});
Also, it's probably good practice to check for error returned.. specifically model.client.query() in this case
I am trying to run a find inside another find and I am not getting any results from the second find operation.
User.find({}, function (err, docs) {
for (i = 0; i < docs.length; i++) {
var tmp = '';
UserGroups.find({userName: docs[i].userName}, function (errin, groups) {
for (g = 0; g < groups.length; g++) {
tmp += ", " + groups[g].groupName;
//console.log(groups[g].groupName);
}
});
console.log(tmp);
//docs[i].group = that;
docs[i].username = decrypt(docs[i].username);
docs[i].password = '';
}
res.render('users', {users: docs});
});
Your UserGroups.find is going to be run asynchronously therefore console.log(tmp) is going to be run before your UserGroups.find has a chance to finish and your call is going to return before you get any results. If you want the results of the UserGroup.find you need to move all of your logic into that callback.
EDIT
This is I believe a far better approach in terms of predictability and query performance. Your previous approach the UserGroup.find being called n number of times. N being the number of users in your database. This approach the database is only queried twice. Once to get all the users and second to get all the groups.
User.find({}, function (err, docs) {
//Get all the usernames before executing the UserGroups query
var userNames = [];
users.forEach(function(element) {
userNames.push(element.userName);
});
UserGroups.find({userName: {$in : userNames}}, function (errin, groups) {
for (var i = 0; i < docs.length; i++) {
//get all the groups that belong to this user
var userGroups = groups.filter(function(value) {
return value.userName === docs[i].userName;
});
var tmp = "";
userGroups.forEach(function(element){
tmp += "," + element.groupName
});
//docs[i].group = that;
docs[i].username = decrypt(user[i].username);
docs[i].password = '';
}
res.render('users', {users: docs});
});
});
Also since it appears you are using Mongoose you can use the built in populate feature in Mongoose to "join" collections together
So let's say I have the following for loop
for(var i = 0; i < array.length; i++){
Model.findOne({ _id = array[i].id}, function(err, found){
//Some stuff
});
}
How do I make this code work? Every time I run it I get array[i] = undefinedbecause the mongo-db query is asynchronous and the loop has already iterated 5 times by the time the first query is even completed. How do I go about tackling this issue and waiting for the query to complete before going on to the next iteration?
This doesn't specifically answer your question, but addresses your problem.
I'd use an $in query and do the filtering all at once. 20 calls to the db is pretty slow compared to 1:
// grab your ids
var arrayIds = myArray.map(function(item) {
return item._id;
});
// find all of them
Model.find({_id: {$in: arrayIds}}, function(error, foundItems) {
if (error) {
// error handle
}
// set up a map of the found ids
var foundItemsMap = {};
foundItems.forEach(function(item) {
foundItemsMap[item._id] = true;
});
// pull out your items that haven't been created yet
var newItems = [];
for (var i = 0; i < myArray.length; i++) {
var arrayItem = myArray[i];
if ( foundItemsMap[arrayItem._id] ) {
// this array item exists in the map of foundIds
// so the item already exists in the database
}
else {
// it doesn't exist, push it into the new array
newItems.push(arrayItem);
}
}
// now you have `newItems`, an array of objects that aren't in the database
});
One of the easiest ways to accomplish something like you want is using promises. You could use the library q to do this:
var Q = require('q');
function fetchOne(id) {
var deferred = Q.defer();
Model.findOne({ _id = id}, function(err, found){
if(err) deferred.reject(err);
else deferred.resolve(found);
});
return deferred.promise;
}
function fetch(ids, action) {
if(ids.length === 0) return;
var id = ids.pop();
fetchOne(id).then(function(model) {
action(model);
fetch(ids, action);
});
}
fetch([1,2,3,4,5], function(model) { /* do something */ });
It is not the most beautiful implementation, but I'm sure you get the picture :)
Not sure if this is the right way, it could be a bit expensive but this how i did it.
I think the trick is to pull all your data and then looking for an id match.
Model.find(function(err, data) {
if (err) //handle it
for (var i=0; i<array.length; i++) {
for (var j=0; ij<data.length; j++) {
if(data[j].id == array[i].id) {
// do something
}
}
}
}