Nested asynchronous mongoDB calls in node js - node.js

I have quite a simple problem, but I can't find an elegant solution to fix this.
In the following code, I have two nested calls to a mongo DB. I use Monk to manage my calls.
The problem is : the for loop (1) loops before the nested insertion can happen. So the next find (2) instruction does not find the last inserted action.
The call order is 1-2-2-2-3-3-3 (for an actionList of size 3). So all my data is inserted.
My objective is to have the call order 1-2-3-2-3-2-3
Do you have any clue of how to manage such a problem, without making a big find on my database and manage my list server-side ? (Get all data, make myself the search, that is quite horrible to do, insert elements I want, then push it all to the db...)
for (var action of actionList)//(1)
{
collectionActions.find(//(2)
{eventid : action.eventid},
function(e,actionsFound)
{
if (actionsFound.length == 0)
{
collectionActions.insert(action, function(err, result)//(3)
{
console.log("insert action : " + action._id);
})
}
}
)
}

The native Promise object has an all method that could be leveraged to help.
Assuming find is a compliant promise, the following code would queue up all of the actions in an array through map and which would return a promise for each action that eventually returns messages to the final then for all.
A couple of notes: your code as it stands swallows all of the errors that might occur (I'm not sure that is want you want); this also assumes that insert returns a promise.
Promise.all([
// Iterate over actionList
actionList.map(function(action) {
// returns a promise with a then already attached
return collectionActions.find({
eventid: action.eventid
}).then(function(e, actionsFound) {
if (actionsFound.length == 0) {
// returns another promise that will resolve up to outer promises
return collectionActions.insert(action, function(err, result) {
// Finally resolve a value for outer promises
return 'insert action : ' + action._id;
});
} else {
// A different value to resolve with if the above promise
// is not required
return 'some other message for ' + action._id;
}
});
})
]).then(function(results) {
// Log out all values resolved by promises
console.log(results);
});
UPDATE: After the clarification of the question it sounds like you just need to chain the promises together rather than run them in parallel.
// Iterate over actionList
actionList.reduce(function(promise, action) {
// Chain promises together
return promise.then(function(results) {
return collectionActions.find({
eventid: action.eventid
}).then(function(e, actionsFound) {
if (actionsFound.length == 0) {
// returns another promise that will resolve up to outer promises
return collectionActions.insert(action, function(err, result) {
// Finally resolve a value for outer promises
return results.push('insert action : ' + action.sourceName);
});
} else {
// A different value to resolve with if the above promise
// is not required
return results.push('some other message for ' + action.sourceName);
}
});
});
}, Promise.resolve([])).then(function(results) {
// Log out all values resolved by promises
console.log(results);
});

I finally got my solution, by using a recursive function.
var currentIndex = 0;
var searchAndInsert = function(actionList)
{
var action = actionList[currentIndex];
if (typeof actionList[currentIndex] != "undefined")
{
collectionActions.find(
{eventid : action.eventid},
function(e,actions)
{
console.log("find ended")
if (actions.length == 0)
{
collectionActions.insert(action, function(err, result)
{
console.log("insert action : " + action.sourceName);
currentIndex++;
if (typeof actionList[currentIndex] != "undefined")
searchAndInsert(actionList);
})
}
else
{
currentIndex++;
if (typeof actionList[currentIndex] != "undefined")
searchAndInsert(actionList);
}
}
)
}
};

Related

Model.findOne() function not working when passed a variable as the value

I am new to node.js.
I am trying to create function, where a randomly generated String is queried to check if it exists or not. If it already exists, the String is randomly generated till it is unique.
let validID = false;
console.log(temp); //temp is the randomly generated String.
while(!validID){
Website.findOne({shortcut: temp},function(err,docs){
if(docs==null){
validID = true;
console.log("The shortcut for the url is" + temp);
}else{
console.log("FOUND");
temp = generateId();
}
});
}
When run, the code is stuck in an infinite while loop.
I tried to see whether the code works with a String value ( not a variable ) passed in as the query inside findOne(). It worked. I am assuming that the fact that temp is a variable is causing the problem. Can variables be passed in as a value in a query? If so, what is the correct way?
Website.findOne operates asynchronously, i.e. the callback-function you passed to it, will be run once the results from the mongodb are fetched. However, node will not be able to actually process this callback, since your callstack never gets emptied due to your while-loop. If you're interested, you can find out more about this here.
One way to solve this is to wrap your Mongo-DB call in a promise, wait for it to resolve, then return if the ID is unique and continue by calling it recursively otherwise (note that this can be highly simplified by using async/await but for understanding how this works using promised are beneficial imo):
function findIdPromise(temp) {
return new Promise((resolve, reject) => {
Website.findOne({
shortcut: temp
}, function (err, docs) {
if (err) {
return reject(err);
}
resolve(docs);
});
});
}
function getNextIsUniqueIdPromise(shortcut) {
return findIdPromise()
.then(docs => {
if (docs == null) {
return shortcut;
}
return getNextIsUniqueIdPromise(generateId());
});
}
// call it initially with
getNextIsUniqueIdPromise(firstShortcutToCheck)
.then(shortcut => {
console.log("The shortcut for the url is" + shortcut):
})
.catch(err => {
console.log("an error occured", err):
});

Avoid callback multi-invocation when forEach is used

I have a function that processes an array of data (first parameter) and, once the procesing is finished, it invokes only one time a callback function (second parameter). I'm using forEach to process data item by item, consisting the processing of each item in some checkings and storing the param in database. The function storeInDB() does the storing work and uses a callback (second parameter) when the item has been stored.
A first approach to the code is the following:
function doWork(data, callback) {
data.forEach(function (item) {
// Do some check on item
...
storeInDB(item, function(err) {
// check error etc.
...
callback();
});
});
}
However, it's wrong, as the the callback function will be invoked several times (as many as element in the data array).
I'd like to know how to refactor my code in order to achieve the desired behaviour, i.e. only one invocation to callback once the storing work is finished. I guess that async could help in this task, but I haven't find the right pattern yet to combine async + forEach.
Any help is appreciated!
You can use a library such as async to do this, although I would recommend using promises if possible. For your immediate problem you can use a counter to determine how many storage calls have completed and call the callback when the total number are completed.
let counter = 0;
data.forEach(function (item) {
// Do some check on item
...
storeInDB(item, function(err) {
// check error etc.
counter++
if (counter == data.length) {
callback();
}
});
});
you can also utilize the three parameters passed to the function to execute on each array method
function doWork(data, callback) {
data.forEach(function (value,idx,arr) {
// Do some check on item
...
storeInDB(arr[idx], function(err) {
// check error etc.
...
if ( (idx + 1) === arr.length ) {
callback();
}
});
});
}
If storeInDB function returns a promise, you can push all async functions to an array and use Promise.all. After all tasks run successfully, It will invokes callback function.
Hope this helps you.
function doWork(data, callback) {
let arr = [];
data.map(function(itm) {
// Do some check on item
...
arr.push(storeInDB(item));
});
Promise.all(arr)
.then(function(res) {
callback();
});
}

Returning response from Mongoose promise

Followup from this question > Stopping response if document isn't found since it was recommended I use Promise.
So basic premise, I want node to return "Can't find ID" message if we can't find the id in our database.
v1.post("/", function(req, res) {
// If the project_id isn't provided, return with an error.
if ( !("project_id" in req.body) ) {
return res.send("You need to provide Project ID");
}
// Check if the Project ID is in the file.
helper.documentExists( ProjectsData, {project_id: req.body.project_id} )
.then(function(c) {
if ( c == 0 ) {
return res.send("The provided Project Id does not exist in our database.");
} else {
var gameDataObj = req.body;
GameData.addGameId(gameDataObj, function (err, doc) {
if (err) {
if (err.name == "ValidationError") {
return res.send("Please send all the required details.");
}
throw err;
};
res.json(doc);
})
};
});
});
And helper.documentExists
module.exports = {
documentExists: function(collection, query) {
return collection.count( query ).exec();
},
};
But the script continues to run after this and prints the "required data not found".
Output:
required data not found
1
I am using native ES6 Promises.
var mongoose = require("mongoose");
mongoose.Promise = global.Promise;
EDIT: Included the entire get route. (will fix those throw err later)
#######POINT 1#########
ProjectsData.count( {project_id: req.body.project_id} )
.then(function(c) {
#######POINT 3#########
if ( c == 0 ) {
console.log("1");
return res.send("The provided Project Id does not exist in our database.");
console.log("2");
}
});
#######POINT 2#########
//some other logic
console.log("required data not found");
Following async workflow: after POINT 1, the promise is created and your handler is attached. Now POINT 2 will continue, while (at some future clock the promise is resolved and you reach POINT 3.
With my limited understanding of your workflow/purpose I'd say simply put POINT 2 code in the else{} of the if at POINT 3 (as you rightly guessed in the comments).
EDIT: thanks to #jfriend00 for pointing out a serious mistake in the previous version of my answer.
Your code essentially results in this:
ProjectsData.count().then(...);
console.log("required data not found");
So, of course the second console.log() is going to run and print. Nothing that happens in the .then() handler runs until long after the console.log() has already run. And, even then, it can't stop other code from running. Promises don't make the interpreter "wait". They just provide structure for you to coordinate your asynchronous operations.
If you want to branch with promises, then you have to branch inside the .then() handler, not after it.
You don't show enough of the rest of what you're doing to know how to recommend a complete solution. We need to see the rest of your request in order to help you with the proper branching based on asynchronous results.
You probably need something like this:
ProjectsData.count( {project_id: req.body.project_id} ).then(function(c) {
if ( c == 0 ) {
return res.send("The provided Project Id does not exist in our database.");
} else {
// put other logic here
}
}).catch(function(err) {
// handle error here
});

Wait for all query to finish and fill at the same time asynchronously

I want to fill each object of the result of a query, with other querys, and I want to do all in asynchronously way
Here is an example of the way how I do actually
var q = knex.select().from('sector');
q.then(function (sectores) {
var i = -1;
(function getDetalles(sectores) {
i++;
if(i < sectores.length){
knex.select().from('sector_detalle')
.where('sector_id', sectores[i].id)
.then(function (detalles) {
// this what i want to do asynchronously
sectores[i].sector_detalles = detalles;
console.log(sectores[i]);
getDetalles(sectores);
});
} else {
res.send({sucess: true, rows: sectores});
}
})(sectores);
});
I do some reserch and found this wait for all promises to finish in nodejs with bluebird
is close to what I want but don't know how to implement
I think you're looking for the map method that works on a promise for an array, and will invoke an asynchronous (promise-returning) callback for each of the items in it:
knex.select().from('sector').map(function(sector) {
return knex.select().from('sector_detalle')
.where('sector_id', sector.id)
.then(function(detalles) {
sector.sector_detalles = detalles;
// console.log(sector);
return sector;
});
}).then(function(sectores) {
res.send({sucess: true, rows: sectores});
});

return value from a function with asynchronous commands

I'm writing a NodeJS v0.10 application with MariaSQL.
i want to create a function that returns the id of a row, and if the row doesn't exist, to create it and then return the id.
this is what I have so far:
TuxDb.prototype.createIfNEDrinkCompany = function(drinkCompany) {
this.client.query("insert into drink_company(drink_company_name) values(:drink_company) on duplicate key update drink_company_id=drink_company_id",
{'drink_company' : drinkCompany})
.on('result',function(res) {
res.on('end',function(info){
if (info.insertId > 0) {
return info.insertId;
} else {
this.client.query("select drink_company_id from drink_company where drink_company_name = :drink_company",{'drink_company' : drinkCompany})
.on('result',function(res){
res.on('row',function(row){
return row.drink_company_id;
});
});
}
});
});
}
now the problem is that since it's asynchronous, the function ends before the value is returned.
how can I resolve this issue ?
The standard way in nodejs of dealing with async code is to provide a callback function as a last argument to your method and call it whenever your asynchronous finishes. The callback function standard signature is (err, data) - you can read more about here: Understanding callbacks in Javascript and node.js
Rewriting your code:
TuxDb.prototype.createIfNEDrinkCompany = function(drinkCompany, callback) {
this.client.query("insert into drink_company(drink_company_name) values(:drink_company) on duplicate key update drink_company_id=drink_company_id",
{'drink_company' : drinkCompany})
.on('result',function(res) {
res.on('end',function(info){
if (info.insertId > 0) {
callback(null, row.drink_company_id);
} else {
this.client.query("select drink_company_id from drink_company where drink_company_name = :drink_company",{'drink_company' : drinkCompany})
.on('result',function(res){
res.on('row',function(row){
callback(null, row.drink_company_id);
});
});
}
});
});
}
and then in the code calling your method
db.createIfNEDrinkCompany(drinkCompany, function(err, id){
// do something with id here
})

Resources