Mongoose reacting differently to presence of callbacks - node.js

I was debugging a chunk of my Mongoose queries that I wanted to execute one after another to avoid inconsistency. After lot of trial, errors and tests, I finally stumbled upon a solution that seemed to work.
In one query, I include a query and in another, I skip including the callback function. The callback I am referring to is seen in the syntax of a Model.findByIdAndUpdate statement.
Model.findByIdAndUpdate(id, [update], [options], [***callback***])
The one which has a callback function executes perfectly and the one which doesn't, doesn't execute. It's encapsulated in a function which returns a mongoose promsise.
Here is the first query with the schema in Robomongo which adds an 'organiser' successfully:
function addUserToHackathonOrganisers(userId, hackathonId) {
return Hackathon.findOneAndUpdate(
{hackathonId: hackathonId},
{$addToSet: {organisers: userId.toString()}},
{new: true}
);
}
Notice organisers Array[0] which indicates no update.
Now, here is the query that works (with the callback in). In the attached image, you can see the organisers Array[1] which indicates that its updated.
function addUserToHackathonOrganisers(userId, hackathonId) {
return Hackathon.findOneAndUpdate(
{hackathonId: hackathonId},
{$addToSet: {organisers: userId.toString()}},
{new: true},
function (err) {
if (err) {
console.log(err);
}
}
);
}
I'm clueless why this happens. Any ideas?

It is because the findOneAndUpdate() method without a callback option returns a Query, it does not execute but the one with a callback executes hence why you see an update with the callback and none without.
From the docs:
A.findOneAndUpdate(conditions, update, options, callback) // executes
A.findOneAndUpdate(conditions, update, options) // returns Query
A.findOneAndUpdate(conditions, update, callback) // executes
A.findOneAndUpdate(conditions, update) // returns Query
A.findOneAndUpdate() // returns Query
If you had written the function as
function addUserToHackathonOrganisers(userId, hackathonId) {
return Hackathon.findOneAndUpdate(
{hackathonId: hackathonId},
{$addToSet: {organisers: userId.toString()}},
{new: true}
).exec(); // chaining exec() to a Query returns a Promise
}
then when you call it, it executes and returns a promise, e.g.
var promise = addUserToHackathonOrganisers(userId, hackathonId);
promise.then(function(addedUser){
console.log(addedUser); // shows the updated user
});
UPDATE
To address your follow-up question in the comments, the callback in the promise's then() method
promise.then(function(addedUser){
console.log(addedUser); // shows the updated user
});
is just the same as the callback in the findOneAndUpdate() option except that instead of two arguments, the promise handles the arguments separately with the error argument caught in the promise's catch() method:
promise.then(function(addedUser){
console.log(addedUser); // shows the updated user
})
.catch(function(err){
console.log(err);
});
For more details, refer to the docs here.

Related

Node.js .map function causing express server to return the response object early

My goal is to create an abstracted POST function for an express running on Node similar to Django's inbuilt REST methods. As a part of my abstracted POST function I'm checking the database (mongo) for valid foreign keys, duplicates, etc..., which is where we are in the process here. The function below is the one that actually makes the first calls to mongo to check that the incoming foreign keys actually exist in the tables/collections that they should exist in.
In short, the inbuilt response functionality inside the native .map function seems to be causing an early return from the called/subsidiary functions, and yet still continuing on inside the called/subsidiary functions after the early return happens.
Here is the code:
const db = require(`../../models`)
const findInDB_by_id = async params => {
console.log(`querying db`)
const records = await Promise.all(Object.keys(params).map(function(table){
return db[table].find({
_id: {
$in: params[table]._ids
}
})
}))
console.log('done querying the db')
// do stuff with records
return records
}
// call await findIndDB_by_id and do other stuff
// eventually return the response object
And here are the server logs
querying db
POST <route> <status code> 49.810 ms - 9 //<- and this is the appropriate response
done querying the db
... other stuff
When I the function is modified so that the map function doesn't return anything, (a) it doesn't query the database, and (b) doesn't return the express response object early. So by modifying this:
const records = await Promise.all(Object.keys(params).map(function(table){
return db[table].find({ // going to delete the `return` command here
_id: {
$in: params[table]._ids
}
})
}))
to this
const records = await Promise.all(Object.keys(params).map(function(table){
db[table].find({ // not returning this out of map
_id: {
$in: params[table]._ids
}
})
}))
the server logs change to:
querying db
done querying the db
... other stuff
POST <route> <status code> 49.810 ms - 9 // <-appropriate reponse
But then I'm not actually building my query, so the query response is empty. I'm experiencing this behavior with the anonymous map function being an arrow function of this format .map(table => (...)) as well.
Any ideas what's going on, why, or suggestions?
Your first version of your code is working as expected and the logs are as expected.
All async function return a promise. So findInDB_by_id() will always return a promise. In fact, they return a promise as soon as the first await inside the function is encountered as the calling code continues to run. The calling code itself needs to use await or .then() to get the resolved value from that promise.
Then, some time later, when you do a return someValue in that function, then someValue becomes the resolved value of that promise and that promise will resolve, allowing the calling code to be notified via await or .then() that the final result is now ready.
await only suspends execution of the async function that it is in. It does not cause the caller to be blocked at all. The caller still has to to deal with a promise returned from the async function.
The second version of your code just runs the db queries open loop with no control and no ability to collect results or process errors. This is often referred to as "fire and forget". The rest of your code continues to execute without regard for anything happening in those db queries. It's highly unlikely that the second version of code is correct. While there are a very few situations where "fire and forget" is appropriate, I will always want to at least log errors in such a situation, but since it appears you want results, this cannot be correct
You should try something like as when it encounter first async function it start executing rest of the code
let promiseArr = []
Object.keys(params).map(function(table){
promiseArr.push(db[table].find({
_id: {
$in: params[table]._ids
}
}))
})
let [records] = await Promise.all(promiseArr)
and if you still want to use map approach
await Promise.all(Object.keys(params).map(async function(table){
return await db[table].find({
_id: {
$in: params[table]._ids
}
})
})
)

Passing a callback to a function that is prefixed with await. Order of excecution

Let's say we have a mongoDB with cars, and users can add cars. The goal is to prevent more than one car with an identical license plate to be saved. Stack is node/express and mongoose.
Scenario: A car with the provided license plate DOES already exist in the db.
First attempt:
console.log('Before check')
await Car.findOne({ licensePlate }, (err, foundCar) => {
if (err) return next(err)
if (foundCar) return res.status(400).send('license plate exists already in db.')
})
console.log('After check')
// some other code, saving to db and stuff
return res.status(200).send('The car was saved succesfully.')
My naive expectation was that if i prefix moongooses findOne function with an await keyword, the code after the "await block" will not be executed because a car is found and then the request is terminated by sending a 400 response. What happens instead is that the code after the "await block" is executed which results in a
Error: Can't set headers after they are sent.
Second attempt:
console.log('Before check')
try {
const foundCar = await Car.findOne({ licensePlate })
if (foundCar) return res.status(400).send('license plate exists already in db.')
} catch (err) {
return next(err)
}
console.log('After check')
// some other code, saving to db and stuff
return res.status(200).send('The car was saved succesfully.')
This works as expected and prevents any code from exceduting after the if(foundCar) check
Question: Can someone enlighten me what happens here in attempt one? Looking at it now it does feel weird to combine a callback with an await keyword, but apperently i don't understand async/await enough to really see what's happening here. Does await in this case mean that "the code waits" only for findOne to finish and then the callback and the rest of the code run at the same time? Any pointer to useful ressources to fully understand what's going on would be very much appreciated.
If the findOne function is not passed a callback function, it returns a Promise object.
If await is invoked with a Promise object, it will wait till the value is resolved by the promise and return the value.
If await is invoked with a non Promise object, it will convert that to a resolved promise.
In the first case, a callback is passed to the findOne, so it is not returning a Promise object and await converts that to a resolved promise. The value of that is resolved in the nextTick of the Node.js execution cycle. So there is no waiting for the results is happening here and even before the callback function is invoked response 200 is sent. When the callback is actually executed, it is trying to send the response headers again and since they are already sent, you are getting this error.
In the second case, findOne actually returns a Promise object. So await waits till the Promise resolves to a value. That is why the second case is working properly.

Using Mongodb variables out of its functions

So I'm making a web application and I'm trying to send variables to an EJS file but when they are sent out of the mongo functions they come out as undefined because it's a different scope for some reason. It's hard to explain so let me try to show you.
router.get("/", function(req, res){
var bookCount;
var userCount;
Books.count({}, function(err, stats){
if(err){
console.log("Books count failed to load.");
}else{
bookCount = stats;
}
});
User.count({}, function(err, count){
if(err){
console.log("User count failed to load.")
}else{
userCount = count;
console.log(userCount);
}
});
console.log(userCount);
//Get All books from DB
Books.find({}, function(err, allbooks){
if(err){
console.log("Problem getting all books");
}else{
res.render("index", {allbooks: allbooks, bookCount: bookCount, userCount: userCount});
}
});
});
So in the User.Count and Books.count I'm finding the number of documents in a collection which works and the number is stored inside of the variables declared at the very top.
After assigning the numbers like userCount i did console.log(userCount) which outputs the correct number which is 3, If was to do console.log(userCount) out of the User.count function it would return undefined, which is a reference to the declaration at the very top.
What is really weird is that Book.Find() has the correct userCount even though its a totally different function. The whole goal im trying to accomplish is doing res.render("index", {userCount: userCount}); outside of the Books.find(). I can do it but of course for some reason it passes undefined instead of 3. I hope this made a shred of sense.
I seem to have found a solution. but if anyone knows a different way I would love to know. So basically all you need to do is move the User.Count function outside of the router.get() function. Not completely sure about the logic of that but it works...
This is a classic asynchronous-operation problem: Your methods (Books.count, Books.find, User.count) are called immediately, but the callback functions you pass to them are not. userCount is undefined in your log because console.log is called before the assignment in the callback function is made. Your code is similar to:
var userCount;
setTimeout(function() {
userCount = 3;
}, 1000);
console.log(userCount); // undefined
User.count takes time to execute before calling back with the result, just like setTimeout takes the specified time to execute before calling its callback. The problem is JS doesn't pause and wait for the timeout to complete before moving on and calling console.log below it, it calls setTimeout, calls console.log immediately after, then the callback function is called one second later.
To render a complete view, you need to be sure you have all of the data before you call res.render. To do so you need to wait for all of the methods to call back before calling res.render. But wait, I just told you that JS doesn't pause and wait, so how can this be accomplished? Promise is the answer. Multiple promises, actually.
It looks like you are using Mongoose models. Mongoose has been written so that if you don't pass a callback function to your methods, they return a promise.
Books.count({}) // returns a promise
JS promises have a method then which takes a callback function that is called when the promise has been resolved with the value of the asynchronous method call.
Books.count({}) // takes some time
.then(function(bookCount) { // called when Books.count is done
// use the bookCount here
})
The problem is, you want to wait for multiple operations to complete, and multiple promises, before continuing. Luckily JS has a utility just for this purpose:
Promise.all( // wait for all of these operations to finish before calling the callback
Books.count({}),
User.count({}),
Books.find({})
)
.then(function(array) { // all done!
// the results are in an array
bookCount = array[0];
userC0unt = array[1];
allBooks = array[2];
})

Bluebird Warning : a promise was created in a handler but was not returned from it

I have an array of filenames which I iterate using this async module for node.
async.eachSeries(imageStore, function(imageDetails,callback){
mongoMan.updateCollection('imageStore',imageDetails,{_id: imageDetails.fileName}).then(function(res){
return callback(null, res);
}).catch(function(err){
logger.error(err);
return callback(err);
});
},function(err){
callback(null);
});
The updateCollection() function is like this:
exports.updateCollection = function(collection, values, condArr){
var result = imageStores.updateAsync(condArr, values, { upsert: true }).then(function(res){
return new Promise.resolve(res);
}).catch(function(err){
logger.error(err);
});
return new Promise.resolve(result);
}
This code works well, updates DB and everything. But I am still unable to resolve the warning that bluebird throws:
Warning: a promise was created in a handler but was not returned from it
at Object.exports.updateCollection (/home/swateek/Documents/codebase/poc/apps/webapp/server/components/mongodb/mongoConn.js:46:22)
at /home/swateek/Documents/codebase/poc/apps/webapp/server/components/imageStore.js:72:24
at /home/swateek/Documents/codebase/poc/apps/webapp/node_modules/async/lib/async.js:181:20
at iterate (/home/swateek/Documents/codebase/poc/apps/webapp/node_modules/async/lib/async.js:262:13)
Have looked up solutionhere but that ain't convincing enough in my case. Atleast is there a way to turn off the warning?
UPDATE
Please check the correct answer below, and here's how my calling function looks like:
function(callback){// post imageStore data to DB
async.eachSeries(imageStore, function(imageDetails,callback){
mongoMan.updateCollection('imageStore',imageDetails,{_id: imageDetails.fileName}).catch(function(err){
logger.error(err);
return callback(err);
});
return callback(null);
},function(err){
callback(null);
});
}
You are asking for trouble and throwing away programming advantages if you try to mix async library callbacks with promises. Pick one structure or the other and use it everywhere. Personally, I'd suggest you move to promises and just "promisify" functions that currently work with callbacks. If you're using the Bluebird Promise library, then it has Promise.promisify() and Promise.promisifyAll() which make it pretty easy to promisify things that use the standard node.js async callback so you can just control everything with Promise logic.
Now, onto your specific issue. That error means that you are creating promises within a .then() handler, but those promises are not returned or chained onto any previous promise. Thus, they are completely independent from your other chain. This is usually a bug (thus the reason for the warning). The only time one might actually want to do that is you have some fire and forget operation that you don't want the outer promise to wait for and you aren't tracking errors on which is not what you have here.
Change to this:
exports.updateCollection = function(collection, values, condArr){
return imageStores.updateAsync(condArr, values, { upsert: true}).catch(function(err){
logger.error(err);
// rethrow error so the caller will see the rejection
throw err;
});
}
Changes:
Return the main promise rather than creating a new promise.
No need for return new Promise.resolve(res); as res is already returned from that promise so you can remove that whole .then() handler as it isn't doing anything.
No need for return new Promise.resolve(result); as you can just return the earlier promise directly.
FYI, though you don't need it here at all, Promise.resolve() can be called directly without new.
You have some issues here:
var result = imageStores.updateAsync(condArr, values, { upsert: true }).then(function(res){
return new Promise.resolve(res);
}).catch(function(err){
logger.error(err);
});
return new Promise.resolve(result);
Firstly: Promise.resolve is not a constructor so it shouldn't be used with new. Secondly: there is no point calling Promise.resolve( result ) at all, just return result which is already a Promise. The intermediate then is pointless as well. You can reduce that code to:
return imageStores.updateAsync(condArr, values, { upsert: true })
.catch( function(err){
logger.error(err);
} )
;

Javascript function using promises and not returning the correct value

As a relative beginning in Javascript development, I'm trying to understand this problem I'm encountering in a function I've built that will be a part of the code to connect to a postgreSQL database.
In this function, I'm using the knex query builder method to check if a table exists in a remote database, and this method resolves to a boolean that indicates whether the string you specified matches with a table of the same name in the database. I've a provided a sample example of the knex syntax so people better understand the function.
knex.schema.hasTable('users').then(function(exists) {
if (!exists) {
return knex.schema.createTable('users', function(t) {
t.increments('id').primary();
t.string('first_name', 100);
t.string('last_name', 100);
t.text('bio');
});
}
});
I am trying to make my function return a boolean by using the .every Array method, which checks each array and should every index pass the condition defined, the .every method should return a true value, otherwise false. I've built a function which takes an array of schema keys, or names of tables, and passes it to the .every method. The .every method then uses the knex.schema.hasTable method to return a true or false.
My concern is that through many different configurations of the function, I have not been able to get it to return a correct value. Not only does it return an incorrect value, which may have something to do with .every, which I believe can return "truthey" values, but after defining the function, I will often get a "Function undefined" error when calling it later in the file. Here is a sample of my function - again I think it is moreso my poor understanding of how returns, promises and closures are working together, but if anyone has insight, it would be much appreciated.
var schemaTables = ['posts','users', 'misc'];
// should return Boolean
function checkTable() {
schemaTables.every(function(key) {
return dbInstance.schema.hasTable(key)
.then(function(exists) {
return exists;
});
});
}
console.log(checkTable(), 'checkTable function outside');
// console.log is returning undefined here, although in other situations,
I've seen it return true or false incorrectly.
Your function is not working properly for two reasons:
You are not returning the in the checkTable function declaration, so it will always return undefined.
You should write:
function checkTable() {
return schemaTables.every(function(key) {
return dbInstance.schema.hasTable(key)
.then(function(exists) {
return exists;
});
});
}
Anyway you will not get what you want just adding return. I'll explain why in the second point.
Array.prototype.every is expecting a truthy or falsey value syncronously but what the dbInstance.schema.hasTable returns is a Promise object (and an object, even if empty, is always truthy).
What you have to do now is checking if the tables exist asynchronously, i'll show you how:
var Promise = require("bluebird");
var schemaTables = ['posts', 'users', 'misc'];
function checkTable(tables, callback) {
// I'm mapping every table into a Promise
asyncTables = tables.map(function(table) {
return dbInstance.schema.hasTable(table)
.then(function(exists) {
if (!exists)
return Promise.reject("The table does not exists");
return Promise.resolve("The table exists");
});
});
// If all the tables exist, Promise.all return a promise that is fulfilled
// when all the items in the array are fulfilled.
// If any promise in the array rejects, the returned promise
// is rejected with the rejection reason.
Promise.all(asyncTables)
.then(function(result) {
// i pass a TRUE value to the callback if all the tables exist,
// if not i'm passing FALSE
callback(result.isFulfilled());
});
}
checkTable(schemaTables, function (result) {
// here result will be true or false, you can do whatever you want
// inside the callback with result, but it will be called ASYNCHRONOUSLY
console.log(result);
});
Notice that as i said before, you can't have a function that returns a true or false value synchronously, so the only thing you can do is passing a callback to checkTable that will execute as soon as the result is ready (when all the promises fulfill or when one of them rejects).
Or you can return Promise.all(asyncTables) and call then on checkTable it self, but i'll leave you this as exercise.
For more info about promises check:
The bluebird website
This wonderful article from Nolan Lawson
Thanks Cluk3 for the very comprehensive answer. I actually solved it myself by using the .every method in the async library. But yes, it was primarily due to both my misunderstanding regarding returns and asynchronous vs synchronous.
var checkTablesExist = function () {
// make sure that all tables exist
function checkTable(key, done) {
dbInstance.schema.hasTable(key)
.then(function(exists) {
return done(exists);
});
}
async.every(schemaTables, checkTable,
function(result) {
return result;
});
};
console.log(checkTablesExist());
// will now print true or false correctly

Resources