how to perform synchronously findOne & create With in foreach Loop? - node.js

In the following code, I have an infinite loop which I don't know why it happens. My best guess is because the function inside is async the loop doesn't wait for it and so the loop never stops. What is the best way to solve this issue ?
for (var i = 0, len = studentsData.length; i < len; i++) {
// (function(i) {
// var p = new Promise(function(resolve, reject) {
// var item = studentsData[i];
// Find student
Student.findOne({
email: item.Email
},
function (err, student) {
if (err) {
reject(err);
} else {
if (!student) {
// Create Student.
var StudentObj = {
firstName: item.Name || null,
lastName: null,
contact: item.Mobile || null,
securityToken: UTIL.randomString(21),
email: item.Email
};
Student.create(StudentObj, function (err, newStudent) {
if (err) {
reject(err);
} else {
// return newStudent;
console.log("Student created with Id=", newStudent._id);
// resolve(newStudent);
}
});
}
}
}
);
//});
// studentPromise.push(p);
// })(i);
}**

Without know what studentsData variable is, I would say i is always less than studentsData.
You are making things very complicated for yourself, trying to run a callback function in a for loop like so.
I would be inclined to rewrite this using a promise and calling all the promises at once with promise.all
You could do something like this:
function findOrCreateStudent(item){
// push all promises into an array called promises
const promises = studentsData.map(async (studentData) => {
// first param is your query, second is the object of the new document you want to create
try {
return await Student.findOrCreate({email: item.Email}, studentData) // I'm assuming studentsData is an array of student objects you want to find or create
}catch(err){
console.log(`Unable to find or create studen ${err}`)
}
})
return Promise.all(promises) // call all promises
}

Related

NodeJS - Nested Promise inside a for loop

I am trying to do a call which retrieves a list of categories. Inside this call I want to loop through the categories and retrieve the items for each category and return them all together. My call retrieves the categories perfectly before I added the loop to retrieve the items.
To double check my call to another controller works, I added a proof of concept block of code which you can see below is commented out. So I know it isn't the call to an external class.
Here is my code:
'use strict';
var mongoose = require('mongoose'),
MenuCategory = mongoose.model('MenuCategory');
module.exports = function(menuItemController) {
var mod = {
listEntireMenu(req, res) {
return new Promise(function(resolve, reject) {
var entireMenu = [];
MenuCategory.find({}, function(err, menuCategories) {
if (err) {
return reject(err)
} else {
//---------------------------
// PROOF OF CONCEPT THAT CALL TO OTHER CONTROLLER WORKS
//---------------------------
//
// var categoryWithItems = menuCategories[0].toObject();
// req.body.menuCategoryID = categoryWithItems._id;
// menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
// if(menuItems)
// {
// return resolve(menuItems);
// }
// else
// {
// return { success: false }
// }
// });
//-----------------------------
for (var i = 0; i < menuCategories.length; i++) {
var categoryWithItems = menuCategories[i].toObject();
var subItems = [];
req.body.menuCategoryID = categoryWithItems._id;
menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
if(menuItems)
{
subItems = menuItems;
}
else
{
return { success: false }
}
});
categoryWithItems.tester = { "itemsList" : subItems };
entireMenu.push(categoryWithItems);
}
return resolve(entireMenu)
}
});
}).then((menuCategories) => {
if(menuCategories)
{
return menuCategories
}
else
{
return { success: false }
}
});
},
}
return mod;
};
What I actually get returned is this :
[
{
"_id": "5ed16fxxxxxxxx95676e37",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category One",
"Created_date": "2020-05-29T20:26:34.991Z",
"__v": 0,
"tester": {
"itemsList": []
}
},
{
"_id": "5ed170xxxxxx95676e38",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category Two",
"Created_date": "2020-05-29T20:26:48.799Z",
"__v": 0,
"tester": {
"itemsList": []
}
}
]
Here is the call from the route.js :
app.get('/api/listEntireMenu', (req, res) => {
menuCategoryController.listEntireMenu(req, res).then(menuCategories => res.json(menuCategories));
})
It never writes the subItems into the object. Is this an async issue or something else? I am not sure how to solve this.
Thanks in advance.
i believe the reason the result of your call to resolve is being returned before the requests are able to complete...for this you need to wait until all the promises or requests have finished properly and returned.
There are two ways you can do this: you could either run them one by one and wait for each one to finish first or run them all concurrently until all of them are done.
Ofcourse the fastest way to do it would be to run them all concurrently so lets go for that way:
so to start, let us not use the for loop and instead remap the iterable array menuCategories to promises of the request, we will use your proof of concept code to make the array of promises
//...
Promise.all(
menuCategories.map((category) => {
let category_with_items = category.toObject();
req.body.menuCategoryID = category_with_items._id;
// here we need to return this since its the promise we are remapping to
return menuItemController.listAllMenuItemsByCategory(req, res)
.then((menuitems) => {
if(menuItems) {
return menuitems;
}
throw 'No menu items found'
});
});
)
// each promise will return menuitems so we have to wait for all the promises to complete
// then with the results of each promise we push the items into the entire menu
.then((itemslist) => {
itemslist.forEach((items) => entireMenu.push(items));
return entireMenu;
})
// lastly we need to handle any errors from the promises
.catch((error) => { success: false });
//...
So now we have...
listEntireMenu(req, res) {
return MenuCategory.find({}, function(err, menuCategories) {
if (err) {
throw err
} else {
entireMenu = [];
return /* the promise all call from above will go right here */;
}
}
I hope it works out, thanks...

How to return response only once after iterating over an array in nodejs?

Here i am iterating an array and save it i database. After completion of iteration, i have to send the response to front end only once(not inside the loop), here is my code. I am declaring a boolean called create quest to false; after iterating all the element in array i have to return response by checking boolean if true? But before completing the loop, below if condition is executing/ so that time boolean is still false; after that line number 9 will start executing.
createQuest = false;
questionList.forEach(element => {
var quest = new Questionnaire({
_id: new mongoose.Types.ObjectId(),
question: element.question,
blockId: blockId
});
quest.save((err1, doc1) => {
9: if(!err1){
createQuest = true;
}else{
res.send({'message':'failed'});
}
})
});
if(createQuest == true){
res.send({'message':'success'});
}
Run multiple parallel task for by iterating over array ( any collection ) and once all of them are finish execute something else
Consider a scenario where you need to Update multiple. You have questions pushed in an array and you want to execute all function dependent of each other.
Solution : Use async.eachSeries()
Here is code to explain same
var async = require('async');
async.eachSeries(questionList,function(element ,eachCb) {
var quest = new Questionnaire({
_id: new mongoose.Types.ObjectId(),
question: element.question,
blockId: blockId
});
quest.save((err1, doc1) => {
if(!err1){
counter++;
}else{
counter++;
return eachCb(err1);
}
eachCb();
})
},function(err,data) {
if(err){
console.log(err);
}
// Once all done, comes here.
});
For Reference Check with
https://codeforgeek.com/asynchronous-programming-in-node-js/
https://dzone.com/articles/how-to-interact-with-a-database-using-the-async-mo
async.eachSeries in node.js
(OR)
Here are both way of saving data with insertMany and save
1) Mongoose save array of documents with insertMany in bulk
db.collection.InsertMany()
var Questionnaire= mongoose.model('Potato', Questionnaire);
write this api in routes directory
router.post('/addDocuments', function (req, res) {
const data = [/* array of object which data need to save in db */];
Questionnaire.insertMany(data)
.then((result) => {
console.log("result ", result);
res.status(200).json({'success': 'new documents added!', 'data': result});
})
.catch(err => {
console.error("error ", err);
res.status(400).json({err});
});
})
BY the way you are implemented you can do the following:
var counter = 0;
questionList.forEach(element => {
var quest = new Questionnaire({
_id: new mongoose.Types.ObjectId(),
question: element.question,
blockId: blockId
});
quest.save((err1, doc1) => {
if(!err1){
counter++;
}else{
counter++;
//res.send({'message':'failed'});
}
})
if(counter == questionList.length) {
res.send({'message':'success'});
}
});
Also you can use promise.all to implement this.
If your purpose is to save all the quests and then send the response then this is what you can do:
let createQuest = false;
let quests = questionList.filter(element => {
return {
_id: new mongoose.Types.ObjectId(),
question: element.question,
blockId: blockId
});
});
Questionnaire.create(quests, function(err) {
if(err)
return res.send({'message':'failed'});
res.send({'message':'success'});
});

How to wait for an asynchronous process to complete inside a for loop before incrementing the loop

I need to iterate through an array. With each iteration, I need to update my database. I need to wait for the first update to be complete and then make the second update.
After searching through several answers, I found ASYNC/AWAIT feature of ES2017. However, I have not been able to implement it so far. The updates are happening randomly and not in a sequence. Please let me know how to implement ASYNC/AWAIT in this situation
Here is my code snippet:
function findRecipe(product, qty) {
return new Promise((resolve, reject) => {
Recipe.findOne({
product: product
}, (err, recipe) => {
if (err) {
reject(err)
} else {
for (let i = 0; i < recipe.items.length; i++) {
Item.findOne({
name: recipe.items[i].name
}, (err, item) => {
if (err) {
reject(err)
} else {
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
item.save((err, item) => {
if (err) {
console.log(err)
} else {
resolve(item)
}
})
}
})
}
}
})
});
}
for (let i = 0; i < bill.product.length; i++) {
//Calling function for updates for each item
findRecipe(bill.product[i], bill.qty[i])
}
It looks like you are nearly there, Just Wrap the loop in a function and make it async.
async function updateAllRecipe(){
for(let i=0;i<bill.product.length;i++){
//Calling function for updates for each item
await findRecipe(bill.product[i],bill.qty[i])
}
}
But seriously though, I think you can leverage of parallelism here using the Promise.All. Is it really necessary to wait for the recipe to finish before queing the next findRecipe method? If not use the promise.all for it to perform faster
Async Await is easy to implement once you know the basic concept of asynchronous nature of Nodejs, I have used for...of loop here which also works asynchronously.
//Async function to perform database updates
async function findRecipe(product, qty) {
try {
let recipe = await Recipe.findOne({ product: product });
let i = 0;
for (itemObj of recipe.items) {
let item = await Item.findOne({ name: itemObj.name });
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
let updatedItem = await item.save();
i++;
}
return true;
}
catch (err) {
return err;
}
}
async function someasyncFunction() {
for (let i = 0; i < bill.product.length; i++) {
//Calling function for updates for each item
await findRecipe(bill.product[i], bill.qty[i])
}
}
Use Promise.all and start process parallel. It'll increase the performance of API.
async function findRecipe(product, qty) {
try {
let recipe = await Recipe.findOne({
product: product
});
const items = await Promise.all(recipe.items.map(itemObj => Item.findOne({
name: itemObj.name
})));
const items = await Promise.all(items.map((item, i) => {
var lessAmt = recipe.quantities[i] * qty;
item.stock -= lessAmt;
return item.save();
}));
return true;
} catch (err) {
return err;
}
}
async function someasyncFunction() {
await Prmise.all(bill.product.map((product, i) => findRecipe(product, bill.qty[i])));
}

returning Mongoose query result from Async call

I'm working on a problem where I need to query the db for an instance of a Voter, and use that instance to update an Election, returning to the original function whether that update was successful or not. My code currently looks like this:
function addCandidatesToElection(req, res) {
let electionName = req.body.electionName;
let candidates = req.body.candidates;
let addedCandidatesSucessfully = true;
for(let i=0; i<candidates.length; i++) {
addedCandidatesSucessfully = _addCandidateToElection(electionName, candidates[i]);
console.log("added candidates sucessfully:" + addedCandidatesSucessfully);
}
if(addedCandidatesSucessfully) {
res.send("createElection success");
} else {
res.send("createElection fail");
}
}
which calls this function:
function _addCandidateToElection(electionName, candidateName) {
async.parallel(
{
voter: function(callback) {
Voter.findOne({ 'name' : candidateName }, function(err,voter) {
callback(err, voter);
});
}
},
function(e, r) {
if(r.voter === null){
return 'Voter not found';
} else {
Election.findOneAndUpdate(
{'name': electionName },
{$push: { candidates: r.voter }},
{new: true},
function(err, election) {
if(err){ return err; }
return (election) ? true : false;
});
}
}
);
}
I've already tried printing out the Voter instance(r.voter) to check if it exists (it does), and also printing out the election object returned by the mongoose call, which also works. However, I'm getting a null value in the
addedCandidatesSucessfully = _addCandidateToElection(electionName, candidates[i]);
line, regardless of the result of the call. I think it has to do with the mongoose call returning a local value which is never returned to the function that called _addCandidateToElection, but I don't know how I should return that. I've tried putting control flags such as
let foundAndUpdatedElection = false;
on the first line of _addCandidateToElection and updating it inside the Mongoose query's callback, but apparently it doesn't change.
How should I return the result of the query to the addCandidatesToElection function?
You should probably 'promisify' your code to help you better deal with the asynchronous nature of js. Try the following instead of your example:
function findVoter(candidateName) {
return new Promise(function(resolve, reject) {
Voter.findOne({ 'name' : candidateName }, function(err,voter) {
if(error) {
reject(error);
} else {
resolve(voter);
}
});
});
}
function addCandidateToElection(electionName, candidateName) {
return findVoter(candidateName).then(function(voter) {
return new Promise(function(resolve, reject) {
Election.findOneAndUpdate(
{'name': electionName },
{$push: { candidates: voter }},
{new: true},
function(err, election) {
if (err) {
reject(err);
} else {
resolve(!!election);
}
});
});
}
function addCandidatesToElection(req, res) {
let electionName = req.body.electionName;
let candidates = req.body.candidates;
let addedCandidatesSucessfully = true;
let candidatePromiseArray = [];
for(let i=0; i<candidates.length; i++) {
candidatePromiseArray.push(addCandidateToElection(electionName, candidates[i]));
}
Promise.all(candidatePromiseArray)
.then(function(results) {
console.log(results);
res.send('create election success');
})
.catch(function(error) {
console.error(error);
res.send('failed');
});
}
You will also no longer need to use the async library because promises are now native in ES6

Call synchronously recursively function in nodejs

I'm developing MEANJS project. Here a function which return array by fetch data recursively. Now, I have the problem that how to get that array.
Problem:-
let users A have properties is {_id:'001',rmUserId:'001'}, B:{_id:'002',rmUserId:'001'}, C {_id:'003',rmUserId:'002'}, D {_id:'003',rmUserId:'001'}, E {_id:'004',rmUserId:'009'}
if user A will login then allUnderUsers array have B,C,D users. That means all users have to follow an own hierarchy.
A
/ \
B D
/
C
Here is my code:-
module.exports.getUnderUsersByRm = function(currentUser, callback) {
try {
var allUnderUsers = [];
function __getUserByRmId(rmId) {
User.find({ rmUserId: rmId, isAlive: true, status: 'active' })
.exec(function(err, users) {
if (err)
return callback(err)
if (users.length > 0) {
users.forEach(function(ele, i) {
allUnderUsers.push(ele);
__getUserByRmId(ele.rmUserId);
});
} else {
return false;
}
})
}
__getUserByRmId(currentUser._id);
} catch (e) {
callback(e)
}
}
Here I need to get allUnderUsers array after all recursive function called.
I have use callback function like:-
....
...
__getUserByRmId(currentUser._id);
callback(null,'done');
.
.
but it throws an error i.e,
Error: Can't set headers after they are sent
.at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:346:11)
at ServerResponse.header (/home/clsah/projects/LMS/node_modules/express/lib/response.js:719:10)
........ .......
If you take advantage of the promises built into the later versions of mongoose and surface a promise interface from your module, you can do this:
Simulated running code here: https://jsfiddle.net/jfriend00/zr6ynmsu/
module.exports.getUnderUsersByRm = function(currentUser) {
function __getUserByRmId(rmId) {
// return promise here
return User.find({ rmUserId: rmId, isAlive: true, status: 'active' }).exec().then(function(users) {
if (users.length > 0) {
let promises = [];
users.forEach(function(ele, i) {
promises.push(__getUserByRmId(ele.rmUserId));
});
// return promise which will chain it to original promise
// this is the key to getting the master promise to wait
// for everything to be done
return Promise.all(promises).then(results => {
// add in previous results
// flatten all the results together into a single array
// and remove empty results
results.unshift(users);
return [].concat.apply([], results.filter(item => item.length > 0));
});
} else {
return [];
}
});
}
return __getUserByRmId(currentUser);
}
And, then you would use it like this:
const someModule = require('yourModule');
someModule.getUnderUsersByRm(someUser).then(results => {
// process all results here
}).catch(err => {
// error here
});
If you still want your callback interface on getUnderUsersByRm, you can still do that (though if you're doing more than a few async calls, it really is worth using promises for all async operations):
module.exports.getUnderUsersByRm = function(currentUser, callback) {
function __getUserByRmId(rmId) {
// return promise here
return User.find({ rmUserId: rmId, isAlive: true, status: 'active' }).exec().then(function(users) {
if (users.length > 0) {
let promises = [];
users.forEach(function(ele, i) {
promises.push(__getUserByRmId(ele.rmUserId));
});
// return promise which will chain it to original promise
// this is the key to getting the master promise to wait
// for everything to be done
return Promise.all(promises).then(results => {
// add in previous results
// flatten all the results together into a single array
// and remove empty results
results.unshift(users);
return [].concat.apply([], results.filter(item => item.length > 0));
});
} else {
return [];
}
});
}
__getUserByRmId(currentUser).then(result => {
callback(null, result);
}).catch(err => {
callback(err);
});
}
If your user tree is circular, then you can protect against an infinite loop by keeping track of all the visited users. You need some sort of unique key that identifies each user. Since I don't know what that is in your program, I will assume the user you are passing in is already an id. Any property that uniquely identifies the user will work in this scheme:
module.exports.getUnderUsersByRm = function(currentUser) {
let visitedUsers = new Set();
function __getUserByRmId(rmId) {
// return promise here
return User.find({ rmUserId: rmId, isAlive: true, status: 'active' }).exec().then(function(users) {
if (users.length > 0) {
let promises = [];
users.forEach(function(ele, i) {
// make sure we aren't already processing this user
// avoid circular loop
let userId = ele.rmUserId;
if (!visitedUsers.has(userId)) {
visitedUsers.add(userId);
promises.push(__getUserByRmId(userId));
}
});
// return promise which will chain it to original promise
// this is the key to getting the master promise to wait
// for everything to be done
return Promise.all(promises).then(results => {
// add in previous results
// flatten all the results together into a single array
// and remove empty results
results.unshift(users);
return [].concat.apply([], results.filter(item => item.length > 0));
});
} else {
return [];
}
});
}
return __getUserByRmId(currentUser);
}
I tried to implement the solution using async based approach. You may find it naive, but it should work.
function __getUserByRmId(rmId, cb) {
var allUnderUsers = [];
User.find({ rmUserId: rmId, isAlive: true, status: 'active' })
.exec(function(err, users) {
async.each(users, function(user, callback){
if (user._id != rmId){
// recursive call
__getUserByRmId(user._id, function(childUsers){
childUsers.forEach(function (childUser) {
allUnderUsers.push(childUser);
});
callback(); //intermediate callback for async call
});
} else { //condition check to avoid infinite loop
allUnderUsers.push(user);
callback(); //intermediate callback for-loop
}
}, function(err){
cb(allUnderUsers); //final callback with result
});
});
}
module.exports.getUnderUsersByRm = function(currentUser, callback) {
__getUserByRmId(currentUser._id, callback)
};
Logically, it should work. Please give a try and let me know, if there are any issue. For now, it returns array containing parent as well. e.g. [A, B, C, D] for your example.
Use async.until and maintain an array of element to be processed
var async = require('async');
module.exports.getUnderUsersByRm = function(currentUser, callback) {
try {
var allUnderUsers = [];
var usersToProcess = [currentUser._id]; // Array to track what was earlier done with recursion
async.until(function() { // Test function for async.until
return usersToProcess.length === 0;
}, function __getUserByRmId(callback2) { // fn for async.until
User.find({
rmUserId: usersToProcess.shift(), // Take first element of array
isAlive: true,
status: 'active'
})
.exec(function(err, users) {
if (err)
return callback2(err)
if (users.length > 0) {
users.forEach(function(ele, i) {
allUnderUsers.push(ele);
usersToProcess.push(ele.rmUserId);
// __getUserByRmId(ele.rmUserId); // To Remove
});
return callback2(); // Always call callback;
} else {
return callback2(); // Earlier: return false; Return some err argument if you want
}
})
}, callback); // Final callback for async.until
} catch (e) {
callback(e);
}
}
I'm pretty sure that you are getting that error because you run __getUserByRmId function (which inside calls the callback function if an error occurs, and then, besides anything that happen you call the callback function again, which may be sending multiple responses to one same request, and so on, setting headers multiple times even when the response was already sent.
I should have post this in the comments but don't have enough reputation to do so

Resources